taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6 . @basedir@/jobd/gc-util-functions.sh
   7
   8 set -e
   9
  10 umask 002
  11 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  12 clean_git_env
  13
  14 # darcs fast-export | git fast-import with error handling
  15 git_darcs_fetch() (
  16         set_utf8_locale
  17         _err1=
  18         _err2=
  19         exec 3>&1
  20         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  21         $(
  22                 exec 4>&3 3>&1 1>&4 4>&-
  23                 {
  24                         _e1=0
  25                         "$cfg_basedir"/bin/darcs-fast-export \
  26                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  27                         echo $_e1 >&3
  28                 } |
  29                 {
  30                         _e2=0
  31                         git_ulimit fast-import \
  32                                 --export-marks="$(pwd)/gfi-marks" \
  33                                 --export-pack-edges="$(pwd)/gfi-packs" \
  34                                 --force 3>&- || _e2=$?
  35                         echo $_e2 >&3
  36                 }
  37         )
  38         EOT
  39         exec 3>&-
  40         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  41         return $?
  42 )
  43
  44 # bzr fast-export | git fast-import with error handling
  45 git_bzr_fetch() (
  46         set_utf8_locale
  47         BZR_LOG=/dev/null
  48         export BZR_LOG
  49         _err1=
  50         _err2=
  51         exec 3>&1
  52         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  53         $(
  54                 exec 4>&3 3>&1 1>&4 4>&-
  55                 {
  56                         _e1=0
  57                         bzr fast-export --plain \
  58                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  59                         echo $_e1 >&3
  60                 } |
  61                 {
  62                         _e2=0
  63                         git_ulimit fast-import \
  64                                 --export-marks="$(pwd)/gfi-marks" \
  65                                 --export-pack-edges="$(pwd)/gfi-packs" \
  66                                 --force 3>&- || _e2=$?
  67                         echo $_e2 >&3
  68                 }
  69         )
  70         EOT
  71         exec 3>&-
  72         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  73         return $?
  74 )
  75
  76 clear_all_objects_and_packs() {
  77         if [ -d objects ]; then
  78                 # make sure the repository is not left broken
  79                 printf '%s\n' 'ref: refs/heads/master' >HEAD || :
  80                 rm -f packed-refs || :
  81                 find -H refs objects -type f -exec rm -f '{}' + >/dev/null 2>&1 || :
  82                 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
  83         fi
  84 }
  85
  86 exit_err=0
  87 exit_objs=0
  88 send_clone_failed() {
  89         trap "" EXIT
  90         # We must now close the .clonelog file that is open on stdout and stderr
  91         exec >/dev/null 2>&1
  92         # It would be nice if git propagated the SIGXFSZ error on up to the shell,
  93         # perhaps it will at some point in the future.  In any case, the only file
  94         # that might be too big would end up in the objects subdirectory.
  95         # Search for any files of size $cfg_max_file_size512 blocks (if set) or
  96         # larger and trigger the too big failure that way as well.
  97         toobig=
  98         if [ "${cfg_max_file_size512:-0}" != "0" ]; then
  99                 toobig="$(find -H objects -type f -size +$(( $cfg_max_file_size512 - 1 )) -print 2>/dev/null |
 100                         head -n 1)" || :
 101         fi
 102         failaddrs="$(config_get owner)" || :
 103         ccadm="${cfg_admincc:-0}"
 104         xfsz_err=""
 105         if [ -n "$toobig" ] || [ "${exit_err:-0}" = "${var_xfsz_err:-999}" ]; then
 106                 ccadm=1
 107                 reposize="$(cd objects && du -sk . | LC_ALL=C awk '{print $1}')" || :
 108                 if [ -n "$reposize" ]; then
 109                         if [ $reposize -lt 5120 ]; then
 110                                 reposize="$reposize KiB"
 111                         else
 112                                 reposize="$(( $reposize / 1024 ))"
 113                                 if [ $reposize -lt 5120 ]; then
 114                                         reposize="$reposize MiB"
 115                                 else
 116                                         reposize="$(( $reposize / 1024 ))"
 117                                         reposize="$reposize GiB"
 118                                 fi
 119                         fi
 120                 fi
 121                 xfsz_err="
 122
 123 The source repository exceeds our maximum allowed repository size."
 124                 clear_all_objects_and_packs
 125         fi
 126         xobjs_err=""
 127         if [ "${exit_objs:-0}" != "0" ]; then
 128                 ccadm=1
 129                 xobjs_err="
 130
 131 The source repository${exit_objs:+ ($exit_objs objects)} exceeds our maximum allowed object limit."
 132                 clear_all_objects_and_packs
 133         fi
 134         if [ -n "$xfsz_err" ] || [ -n "$xobjs_err" ]; then
 135                 # Mark as an exceeds limit clone failure and remember the exceeds
 136                 # message(s) in both .clone_failed_exceeds_limit and .clonelog
 137                 >.clone_failed_exceeds_limit
 138                 if [ -n "$xfsz_err" ]; then
 139                         [ -z "$reposize" ] ||
 140                         printf '%s\n' "repository size at failure: $reposize" >>.clone_failed_exceeds_limit
 141                         printf '%s\n' "${xfsz_err#??}" >>.clone_failed_exceeds_limit
 142                         printf '%s\n' "${xfsz_err#?}" >>.clonelog
 143                 fi
 144                 if [ -n "$xobjs_err" ]; then
 145                         printf '%s\n' "${xobjs_err#??}" >>.clone_failed_exceeds_limit
 146                         printf '%s\n' "${xobjs_err#?}" >>.clonelog
 147                 fi
 148                 # Remove the .clone_failed file to prevent "restarting" the clone since
 149                 # restarting it will not cure the fact that it exceeds allowed limits
 150                 # And the .clone_in_progress file has to go at the same time
 151                 rm -f .clone_in_progress .clone_failed
 152         fi
 153         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 154         [ "$ccadm" = "0" ] || [ -z "$cfg_admin" ] ||
 155         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
 156         [ -z "$failaddrs" ] ||
 157         {
 158               cat <<EOT
 159 Condolences. The clone of project $proj just failed.$xfsz_err$xobjs_err
 160
 161         * Source URL: $url
 162         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 163
 164 The project settings link may be used to adjust the settings
 165 and restart the clone in order to try the clone again.
 166 EOT
 167                 if [ -f .clonelog ] && [ -r .clonelog ]; then
 168                         echo ""
 169                         echo "Log follows:"
 170                         echo ""
 171                         loglines=$(LC_ALL=C wc -l <.clonelog)
 172                         if [ $loglines -le 203 ]; then
 173                                 cat .clonelog
 174                         else
 175                                 head -n 100 .clonelog
 176                                 echo ""
 177                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
 178                                 echo ""
 179                                 tail -n 100 .clonelog
 180                         fi
 181                 fi
 182         } | mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 183 }
 184
 185 # removes any git-svn leftovers
 186 cleanup_git_svn_leftovers() {
 187
 188         # Remove any stale git-svn temp files
 189         # The git-svn process creates temp files with random 10 character names
 190         # in the root of $GIT_DIR.  Unfortunately they do not have a recognizable
 191         # prefix, so we just have to kill any files with a 10-character name.
 192         # All characters are chosen from
 193         # [A-Za-z0-9_] so we can at least check that and fortunately the only
 194         # collision is 'FETCH_HEAD' but that doesn't matter.
 195         # There may also be temp files with a Git_ prefix as well.
 196         _randchar='[A-Za-z0-9_]'
 197         _randchar2="$_randchar$_randchar"
 198         _randchar4="$_randchar2$_randchar2"
 199         _randchar10="$_randchar4$_randchar4$_randchar2"
 200         find -L . -maxdepth 1 -type f -name "$_randchar10" -exec rm -f '{}' + || :
 201         find -L . -maxdepth 1 -type f -name "Git_*" -exec rm -f '{}' + || :
 202 }
 203
 204 # removes all leftovers from a previous failed clone attempt
 205 cleanup_failed_clone() {
 206
 207         # Remove any left-over svn-remote.svn or remote.origin config
 208         git config --remove-section svn-remote.svn 2>/dev/null || :
 209         git config --remove-section remote.origin 2>/dev/null || :
 210
 211         # If there is a remote-template.origin section, pre-seed the
 212         # remote.origin section with its contents
 213         git config --get-regexp '^remote-template\.origin\..' |
 214         while read name value; do
 215                 if [ -n "$name" ] && [ -n "$value" ]; then
 216                         git config "remote${name#remote-template}" "$value"
 217                 fi
 218         done
 219
 220         # Any pre-existing FETCH_HEAD from a previous clone failed or not is
 221         # now garbage to be removed
 222         rm -f FETCH_HEAD
 223
 224         # Remove any stale ref locks
 225         clear_stale_ref_locks
 226
 227         # Remove any left-over svn dir from a previous failed attempt
 228         rm -rf svn
 229
 230         # Remove any left-over .darcs dirs from a previous failed attempt
 231         rm -rf *.darcs
 232
 233         # Remove any left-over repo.hg dir from a previous failed attempt
 234         rm -rf repo.hg
 235
 236         # Remove any left-over import/export/temp files from a previous failed attempt
 237         rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks* hg2git-state \
 238                 gfi-marks gfi-packs .pkts-temp .refs-temp
 239
 240         # Remove any git-svn junk
 241         cleanup_git_svn_leftovers
 242
 243         # We want a gc right after the clone, so re-enable that just in case.
 244         # There's a potential race where we could add it and gc.sh could remove
 245         # it, but we'll reunset lastgc just before we remove .delaygc at the end.
 246         [ -e .delaygc ] || >.delaygc
 247         git config --unset gitweb.lastgc 2>/dev/null || :
 248
 249         # Remove all pre-existing refs
 250         rm -f packed-refs
 251         git for-each-ref --format='delete %(refname)' | git_updateref_stdin 2>/dev/null || :
 252
 253         # The initial state before a clone starts has HEAD as a symbolic-ref to master
 254         git symbolic-ref HEAD refs/heads/master
 255
 256         # HEAD is no longer "ok"
 257         git config --unset girocco.headok 2>/dev/null || :
 258
 259         # We, perhaps, ought to remove any packs/loose objects now, but the next gc
 260         # will get rid of any extras.  Also, if we're recloning the same thing, any
 261         # preexisting packs/loose objects containing what we're recloning will only
 262         # speed up the reclone by avoiding some disk writes.  So we don't kill them.
 263
 264         # It's just remotely possible that a bunch of failures in a row could
 265         # create a big mess that just keeps growing and growing...
 266         # Trigger a .needsgc if that happens.
 267         check_and_set_needsgc
 268 }
 269
 270 proj="${1%.git}"
 271 cd "$cfg_reporoot/$proj.git"
 272 bang_reset
 273
 274 ! [ -e .delaygc ] || >.allowgc || :
 275
 276 trap "exit_err=$?; echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 277 echo "Project: $proj"
 278 echo "   Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
 279 echo ""
 280 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 281 url="$(config_get baseurl)" || :
 282 case "$url" in *"       "*|*" "*|"")
 283         echo "Bad mirror URL (\"$url\")"
 284         exit 1
 285 esac
 286
 287 cleanup_failed_clone
 288
 289 # Record original mirror type for use by update.sh
 290 mirror_type="$(get_url_mirror_type "$url")"
 291 git config girocco.mirrortype "$mirror_type"
 292
 293 echo "Mirroring from URL \"$url\""
 294 echo ""
 295
 296 if [ "$cfg_project_owners" = "source" ]; then
 297         config set owner "$(ls -ldH "${url#file://}" 2>/dev/null | LC_ALL=C awk '{print $3}')"
 298 fi
 299
 300 mailaddrs="$(config_get owner)" || :
 301 [ -z "$cfg_admin" ] ||
 302 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 303
 304 # Make sure we don't get any unwanted loose objects
 305 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
 306 # tweak its configuration to prevent that
 307 git_add_config 'fetch.unpackLimit=1'
 308 # Note the git config documentation is wrong
 309 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
 310 git_add_config 'transfer.unpackLimit=1'
 311 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
 312 git_add_config 'fastimport.unpackLimit=0'
 313
 314 # Initial mirror
 315 echo "Initiating mirroring..."
 316 headref=
 317 showheadwarn=
 318 warnempty=
 319
 320 # remember the starting time so we can easily combine fetched loose objects
 321 # we sleep for 1 second after creating .needspack to make sure all objects are newer
 322 if ! [ -e .needspack ]; then
 323         rm -f .needspack
 324         >.needspack
 325         sleep 1
 326 fi
 327
 328 case "$url" in
 329         svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 330                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 331                 # Allow the username to be specified in the "svn-credential.svn.username"
 332                 # property and the password in the "svn-credential.svn.password" property
 333                 # Use an 'anonsvn' username by default as is commonly used for anonymous svn
 334                 # Default the password to the same as the username
 335                 # The password property will be ignored unless a username has been specified
 336                 if svnuser="$(git config --get svn-credential.svn.username)" && [ -n "$svnuser" ]; then
 337                         if ! svnpass="$(git config --get svn-credential.svn.password)"; then
 338                                 svnpass="$svnuser"
 339                         fi
 340                         url1="${url#*://}"
 341                         url1="${url1%%/*}"
 342                         case "$url1" in ?*"@"?*)
 343                                 urlsch="${url%%://*}"
 344                                 url="$urlsch://${url#*@}"
 345                         esac
 346                 else
 347                         # As a fallback, check in the URL, just in case
 348                         url1="${url#*://}"
 349                         url1="${url1%%/*}"
 350                         svnuser=
 351                         case "$url1" in ?*"@"?*)
 352                                 urlsch="${url%%://*}"
 353                                 url="$urlsch://${url#*@}"
 354                                 url1="${url1%%@*}"
 355                                 svnuser="${url1%%:*}"
 356                                 if [ -n "$svnuser" ]; then
 357                                         svnpass="$svnuser"
 358                                         case "$url1" in *":"*)
 359                                                 svnpass="${url1#*:}"
 360                                         esac
 361                                 fi
 362                         esac
 363                         if [ -z "$svnuser" ]; then
 364                                 svnuser="anonsvn"
 365                                 svnpass="anonsvn"
 366                         fi
 367                 fi
 368                 GIT_ASKPASS_PASSWORD="$svnpass"
 369                 export GIT_ASKPASS_PASSWORD
 370                 # We just remove svn+ here, so svn+http://... becomes http://...
 371                 # We also remove a trailing '/' to match what git-svn will do
 372                 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
 373                 svnurl="${svnurl%/}"
 374                 # We require svn info to succeed on the URL otherwise it's
 375                 # simply not a valid URL and without using -s on the init it
 376                 # will not otherwise be tested until the fetch
 377                 svn --non-interactive --username "$svnuser" --password "$svnpass" info "$svnurl" >/dev/null
 378                 # We initially use -s for the init which will possibly shorten
 379                 # the URL.  However, the shortening can fail if a password is
 380                 # not required for the longer version but is for the shorter,
 381                 # so try again without -s if the -s version fails.
 382                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 383                 # create a .git subdirectory!
 384                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" -s "$svnurl" <"$mtlinesfile" ||
 385                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" "$svnurl" <"$mtlinesfile"
 386                 # We need to remember this url so we can detect changes because
 387                 # ever so "helpful" git-svn may shorten it!
 388                 config_set svnurl "$svnurl"
 389                 # At this point, since we asked for a standard layout (-s) git-svn
 390                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 391                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 392                 # config items.  We could avoid this by not using the '-s' option
 393                 # but then we might not get all the history.  If, for example, we
 394                 # are cloning an http://svn.example.com/repos/public repository that
 395                 # early in its history moved trunk => public/trunk we would miss that
 396                 # earlier history without allowing the funky shorten+prefix behavior.
 397                 # So we read back the svn-remote.svn.fetch configuration and compute
 398                 # the prefix.  This way we are sure to get the correct prefix.
 399                 gitsvnurl="$(git config --get svn-remote.svn.url)" || :
 400                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1)" || :
 401                 gitsvnprefix="${gitsvnfetch%%:*}"
 402                 gitsvnsuffix="${gitsvnprefix##*/}"
 403                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 404                 # Ask git-svn to store everything in the normal non-remote
 405                 # locations being careful to use the correct prefix
 406                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 407                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 408                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 409                 # look for additional non-standard directories to fetch
 410                 # check for standard layout at the same time
 411                 foundstd=
 412                 foundfile=
 413                 svn --non-interactive --username "$svnuser" --password "$svnpass" ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null |
 414                 { while read file; do case $file in
 415                         # skip the already-handled standard ones and any with a space or tab
 416                         *'      '*|*' '*) :;;
 417                         trunk/|branches/|tags/) foundstd=1;;
 418                         # only fetch extra directories from the $svnurl root (not any files)
 419                         *?/) git config --add svn-remote.svn.fetch \
 420                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 421                         *?) foundfile=1;;
 422                 esac; done
 423                 # if files found and no standard directories present use a simpler layout
 424                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 425                         git config --unset svn-remote.svn.branches
 426                         git config --unset svn-remote.svn.tags
 427                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 428                 fi; }
 429                 test $? -eq 0
 430                 # git svn fetch on a very large repo can take some time and the
 431                 # remote server may interrupt the connection from time to time.
 432                 # keep retrying (after a brief pause) as long as we are making progress.
 433                 # however, we do limit the total number of retries to 1000
 434                 # we will, however, retry up to 5 times even if we're not making progress
 435                 v_get_svn_progress_fingerprint() {
 436                         eval "$1="'"$({ GIT_DIR=. git svn info <"$mtlinesfile" 2>&1; git show-ref --head 2>&1; } |
 437                         git hash-object -t blob --stdin )"' || :
 438                 }
 439                 svn_ret_err() { return "${1:-1}"; }
 440                 svn_retries=1000 # maximum possible fetch attempts no matter what
 441                 svn_retry_backoff_start_half=60 # min retry wait is double this amount in seconds
 442                 svn_backoff_count=7 # max retry wait is $svn_retry_backoff_start_half * 2^$svn_backoff_count
 443                 # Cumulative backoff wait before giving up on consecutive no-progress retries
 444                 #   is approximately 2 * $svn_retry_backoff_start_half * 2^$svn_backoff_count
 445                 # For a $svn_backoff_count of 7 that works out to be exactly 4h14m
 446                 svn_progress=
 447                 v_get_svn_progress_fingerprint svn_progress
 448                 svn_progress_retries="$svn_retries"
 449                 svn_retry_backoff="$svn_retry_backoff_start_half"
 450                 svn_err=0
 451                 while [ "$svn_retries" -gt 0 ]; do
 452                         svn_retries="$(( $svn_retries - 1 ))"
 453                         svn_err=0
 454                         GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1
 455                         export GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 456                         unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 457                         saveconfig="$GIT_CONFIG_PARAMETERS"
 458                         git_add_config 'gc.auto=1'
 459                         git_add_config 'gc.autoPackLimit=1'
 460                         # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 461                         GIT_DIR=. git_ulimit svn fetch --log-window-size=$var_log_window_size --username="$svnuser" --quiet <"$mtlinesfile" || svn_err="$?"
 462                         GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 463                         export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 464                         unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 465                         unset GIT_CONFIG_PARAMETERS
 466                         [ -z "$saveconfig" ] || {
 467                                 GIT_CONFIG_PARAMETERS="$saveconfig"
 468                                 export GIT_CONFIG_PARAMETERS
 469                         }
 470                         [ "${svn_err:-1}" -ne 0 ] || break # success!
 471                         # Check to see if we made any progress
 472                         v_get_svn_progress_fingerprint svn_progress_now
 473                         if [ "$svn_progress_now" != "$svn_progress" ]; then
 474                                 # we made progress, continue the loop with min wait
 475                                 svn_progress="$svn_progress_now"
 476                                 svn_progress_retries="$svn_retries"
 477                                 svn_retry_backoff="$svn_retry_backoff_start_half"
 478                         else
 479                                 # no progress, but we only give up after
 480                                 # $svn_backoff_count no-progress attempts in a row
 481                                 [ "$(( $svn_progress_retries - $svn_retries ))" -lt "$svn_backoff_count" ] ||
 482                                         break # failure
 483                                 # continue but only after twice the previous wait
 484                                 # (which will still be the min wait if this is the
 485                                 # first no-progress retry after making some progress)
 486                         fi
 487                         svn_retry_backoff="$(( 2 * $svn_retry_backoff ))"
 488                         # Pause for $svn_retry_backoff seconds before retrying to be friendly to the server
 489                         # Use that time to pack up loose objects if there are "lotsa" them
 490                         if ! lotsa_loose_objects_or_sopacks; then
 491                                 echo "Pausing for $svn_retry_backoff seconds before retrying ($(date))"
 492                                 sleep "$svn_retry_backoff"
 493                         else
 494                                 pausestop="$(( $(date '+%s') + $svn_retry_backoff ))"
 495                                 echo "Pausing and packing loose objects for $svn_retry_backoff seconds before retrying ($(date))"
 496                                 pack_incremental_loose_objects_if_lockable ||
 497                                 echo "Packing skipped (only pausing): $lockerr"
 498                                 timenow="$(date '+%s')"
 499                                 if [ "$timenow" -lt "$pausestop" ]; then
 500                                         sleepamt="$(( $pausestop - $timenow ))"
 501                                         [ "$sleepamt" -le "$svn_retry_backoff" ] ||
 502                                                 sleepamt="$svn_retry_backoff" # paranoia check
 503                                         sleep "$sleepamt"
 504                                 fi
 505                         fi
 506                         cleanup_git_svn_leftovers
 507                         echo "Retrying fetch ($(date))"
 508                 done
 509                 [ "${svn_err:-1}" -eq 0 ] || svn_ret_err "$svn_err"
 510                 test ${svn_err:-1} -eq 0
 511                 # git svn does not preserve group permissions in the svn subdirectory
 512                 chmod -R ug+rw,o+r svn
 513                 # git svn also leaves behind ref turds that end with @nnn
 514                 # We get rid of them now
 515                 git for-each-ref --format='%(refname)' |
 516                 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
 517                 git_updateref_stdin
 518                 unset GIT_ASKPASS_PASSWORD
 519                 ;;
 520         darcs://* | darcs+http://* | darcs+https://*)
 521                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 522                 case "$url" in
 523                         darcs://*) darcsurl="http://${url#darcs://}";;
 524                         *) darcsurl="${url#darcs+}";;
 525                 esac
 526                 git_darcs_fetch "$darcsurl"
 527                 ;;
 528         bzr://*)
 529                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 530                 # we just remove bzr:// here, a typical bzr url is just
 531                 # "lp:foo"
 532                 bzrurl="${url#bzr://}"
 533                 git_bzr_fetch "$bzrurl"
 534                 ;;
 535         hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
 536                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 537                 # We just remove hg+ here, so hg+http://... becomes http://...
 538                 hgurl="${url#hg+}"
 539                 # Perform the initial hg clone
 540                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 541                 # Do the fast-export | fast-import
 542                 git_hg_fetch
 543                 ;;
 544         *)
 545                 # We manually add remote.origin.url and remote.origin.fetch
 546                 # to simulate a `git remote add --mirror=fetch` since that's
 547                 # not available until Git 1.7.5 and this way we guarantee we
 548                 # always get exactly the intended configuration and nothing else.
 549                 git config remote.origin.url "$url"
 550                 if ! is_gfi_mirror_url "$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
 551                         git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
 552                         git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
 553                         git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
 554                         git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
 555                         git config --bool girocco.lastupdateclean true
 556                 else
 557                         git config --replace-all remote.origin.fetch "+refs/*:refs/*"
 558                         git config --bool girocco.lastupdateclean false
 559                 fi
 560                 # Set the correct HEAD symref by using ls-remote first
 561                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp ||
 562                 {
 563                         # Since everything was redirected, on failure there'd be no output,
 564                         # so let's make some failure output
 565                         cat .pkts-temp
 566                         echo ""
 567                         echo "git ls-remote \"$url\" failed"
 568                         exit 1
 569                 }
 570                 # Compensate for git() {} side effects
 571                 unset GIT_TRACE_PACKET
 572                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 573                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 574                 # out of the packet trace data.
 575                 if [ -s .refs-temp ]; then
 576                         # Nothing to do unless the remote repository has at least 1 ref
 577                         # See if we got a HEAD ref
 578                         head="$(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+HEAD\$" <.refs-temp | LC_ALL=C awk '{print $1}')"
 579                         # If the remote has HEAD set to a symbolic ref that does not exist
 580                         # then we will not receive a HEAD ref in the ls-remote output
 581                         headref=
 582                         showheadwarn=
 583                         symrefcap=
 584                         if [ -n "$head" ]; then
 585                                 symrefcap="$(LC_ALL=C sed -ne <.pkts-temp \
 586                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 587                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 588                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 589                                 # matches HEAD otherwise take the first refs/heads/... match
 590                                 matchcnt=0
 591                                 while read ref; do
 592                                         [ -n "$ref" ] || continue
 593                                         matchcnt=$(( $matchcnt + 1 ))
 594                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 595                                                 headref="$ref"
 596                                         fi
 597                                         if [ "$headref" = "${symrefcap:-refs/heads/master}" ] && [ $matchcnt -gt 1 ]; then
 598                                                 break
 599                                         fi
 600                                 done <<-EOT
 601                                 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 602                                         LC_ALL=C awk '{print $2}')
 603                                 EOT
 604                                 # Warn if there was more than one match and $symrefcap is empty
 605                                 # or $symrefcap is not the same as $headref since our choice might
 606                                 # differ from the source repository's HEAD
 607                                 if [ $matchcnt -ge 1 ] && [ "$symrefcap" != "$headref" ] &&
 608                                    { [ -n "$symrefcap" ] || [ $matchcnt -gt 1 ]; }; then
 609                                         showheadwarn=1
 610                                 fi
 611                         fi
 612                         if [ -z "$headref" ]; then
 613                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 614                                 # if it exists otherwise take the first refs/heads/...
 615                                 # We do not support having a detached HEAD.
 616                                 # We always warn now because we will be setting HEAD differently
 617                                 # than the source repository had HEAD set
 618                                 showheadwarn=1
 619                                 while read ref; do
 620                                         [ -n "$ref" ] || continue
 621                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 622                                                 headref="$ref"
 623                                         fi
 624                                         [ "$headref" != "refs/heads/master" ] || break
 625                                 done <<-EOT
 626                                 $(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 627                                         LC_ALL=C awk '{print $2}')
 628                                 EOT
 629                         fi
 630                         # If we STILL do not have a HEAD ref (perhaps the source repository
 631                         # contains only tags) then use refs/heads/master.  It will be invalid
 632                         # but is no worse than we used to do by default and we'll warn about
 633                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 634                         [ -n "$headref" ] || headref="refs/heads/master"
 635                         git symbolic-ref HEAD "$headref"
 636                         pruneopt=--prune
 637                         [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
 638                         # remember the starting time so we can easily detect new packs for fast-import mirrors
 639                         # we sleep for 1 second after creating .gfipack to make sure all packs are newer
 640                         if is_gfi_mirror_url "$url" && ! [ -e .gfipack ]; then
 641                                 rm -f .gfipack
 642                                 >.gfipack
 643                                 sleep 1
 644                         fi
 645                         GIT_SSL_NO_VERIFY=1 git_ulimit remote update $pruneopt
 646                         if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
 647                                 find -L objects/pack -type f -newer .gfipack -name "pack-$octet20*.pack" -print >>gfi-packs
 648                                 rm -f .gfipack
 649                         fi
 650                 else
 651                         warnempty=1
 652                         git symbolic-ref HEAD "refs/heads/master"
 653                 fi
 654                 rm -f .refs-temp .pkts-temp
 655                 ;;
 656 esac
 657
 658 # For systems that do not properly implement the file size limit,
 659 # perform a check here just in case.  Unfortunately by this time
 660 # the excess space has already been used, but at least it will
 661 # be reclaimed almost immediately if we detect an overage here.
 662 if [ "${cfg_max_file_size512:-0}" != "0" ]; then
 663         toobig="$(find -H objects -type f -size +$(( $cfg_max_file_size512 - 1 )) -print 2>/dev/null |
 664                 head -n 1)" || :
 665         if [ -n "$toobig" ]; then
 666                 exit 1 # fail the clone
 667         fi
 668 fi
 669
 670 # Check the max_clone_objects setting now (if set)
 671 if [ "${cfg_max_clone_objects:-0}" != "0" ]; then
 672         objcount="$(git count-objects -v | LC_ALL=C awk 'BEGIN{v=0}/^count:/||/^in-pack:/{v+=$2}END{print v}')" || :
 673         if [ -n "$objcount" ] && [ "$objcount" -gt "$cfg_max_clone_objects" ]; then
 674                 exit_objs="$objcount"
 675                 exit 1 # fail the clone
 676         fi
 677 fi
 678
 679 # The objects subdirectories permissions must be updated now.
 680 # In the case of a dumb http clone, the permissions will not be correct
 681 # (missing group write) despite the core.sharedrepository=1 setting!
 682 # The objects themselves seem to have the correct permissions.
 683 # This problem appears to have been fixed in the most recent git versions.
 684 perms=g+w
 685 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 686 chmod $perms $(find -L objects -maxdepth 1 -type d) 2>/dev/null || :
 687
 688 # We may have just cloned a lot of refs and they will all be
 689 # individual files at this point.  Let's pack them now so we
 690 # can have better performance right from the start.
 691 git pack-refs --all
 692
 693 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 694 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 695 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 696 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 697         --count=1 refs/heads >info/lastactivity || :
 698 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 699
 700 # Don't leave a multi-megabyte useless FETCH_HEAD behind
 701 rm -f FETCH_HEAD
 702
 703 # Last ditch attempt to get a valid HEAD for a non-git source
 704 check_and_set_head || :
 705
 706 # The rest
 707 echo "Final touches..."
 708 git update-server-info
 709 trap "" EXIT
 710
 711 # run gc now unless the clone is empty
 712 if [ -z "$warnempty" ]; then
 713         git config --unset gitweb.lastgc 2>/dev/null || :
 714         rm -f .delaygc .allowgc
 715 fi
 716
 717 emptynote=
 718 [ -z "$warnempty" ] ||
 719 emptynote="
 720 WARNING:  You have mirrored an empty repository.
 721 "
 722 headnote=
 723 [ -z "$showheadwarn" ] || [ -z "$headref" ] ||
 724 headnote="
 725 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 726        Use the \"Project settings\" link to choose a different HEAD symref.
 727 "
 728 sizenote=
 729 ! is_gfi_mirror ||
 730 sizenote="
 731 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 732        size may be somewhat larger than necessary.  This will be corrected
 733        shortly.  If you intend to clone this repository you may want to
 734        wait up to 1 hour before doing so in order to receive the more
 735        compact final size.
 736 "
 737 [ -z "$mailaddrs" ] ||
 738 mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 739 Congratulations! The clone of project $proj just completed.
 740
 741         * Source URL: $url
 742         * GitWeb interface: $cfg_gitweburl/$proj.git
 743         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 744 $emptynote$headnote$sizenote
 745 Have a lot of fun.
 746 EOT
 747
 748 echo "Mirroring finished successfuly!"
 749 # In case this is a re-mirror, lastgc could have been set already so clear it now
 750 git config --unset gitweb.lastgc || :
 751 rm .clone_in_progress
 752 echo "$sizenote@OVER@"