taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6 . @basedir@/jobd/gc-util-functions.sh
   7
   8 set -e
   9
  10 umask 002
  11 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  12 clean_git_env
  13
  14 # darcs fast-export | git fast-import with error handling
  15 git_darcs_fetch() (
  16         set_utf8_locale
  17         _err1=
  18         _err2=
  19         exec 3>&1
  20         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  21         $(
  22                 exec 4>&3 3>&1 1>&4 4>&-
  23                 {
  24                         _e1=0
  25                         "$cfg_basedir"/bin/darcs-fast-export \
  26                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  27                         echo $_e1 >&3
  28                 } |
  29                 {
  30                         _e2=0
  31                         git fast-import \
  32                                 --export-marks="$(pwd)/gfi-marks" \
  33                                 --export-pack-edges="$(pwd)/gfi-packs" \
  34                                 --force 3>&- || _e2=$?
  35                         echo $_e2 >&3
  36                 }
  37         )
  38         EOT
  39         exec 3>&-
  40         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  41         return $?
  42 )
  43
  44 # bzr fast-export | git fast-import with error handling
  45 git_bzr_fetch() (
  46         set_utf8_locale
  47         BZR_LOG=/dev/null
  48         export BZR_LOG
  49         _err1=
  50         _err2=
  51         exec 3>&1
  52         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  53         $(
  54                 exec 4>&3 3>&1 1>&4 4>&-
  55                 {
  56                         _e1=0
  57                         bzr fast-export --plain \
  58                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  59                         echo $_e1 >&3
  60                 } |
  61                 {
  62                         _e2=0
  63                         git fast-import \
  64                                 --export-marks="$(pwd)/gfi-marks" \
  65                                 --export-pack-edges="$(pwd)/gfi-packs" \
  66                                 --force 3>&- || _e2=$?
  67                         echo $_e2 >&3
  68                 }
  69         )
  70         EOT
  71         exec 3>&-
  72         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  73         return $?
  74 )
  75
  76 send_clone_failed() {
  77         trap "" EXIT
  78         # We must now close the .clonelog file that is open on stdout and stderr
  79         exec >/dev/null 2>&1
  80         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
  81         failaddrs="$(config_get owner)" || :
  82         [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
  83         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
  84         [ -z "$failaddrs" ] ||
  85         {
  86               cat <<EOT
  87 Condolences. The clone of project $proj just failed.
  88
  89         * Source URL: $url
  90         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
  91
  92 The project settings link may be used to adjust the settings
  93 and restart the clone in order to try the clone again.
  94 EOT
  95                 if [ -f .clonelog ] && [ -r .clonelog ]; then
  96                         echo ""
  97                         echo "Log follows:"
  98                         echo ""
  99                         loglines=$(LC_ALL=C wc -l <.clonelog)
 100                         if [ $loglines -le 203 ]; then
 101                                 cat .clonelog
 102                         else
 103                                 head -n 100 .clonelog
 104                                 echo ""
 105                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
 106                                 echo ""
 107                                 tail -n 100 .clonelog
 108                         fi
 109                 fi
 110         } | mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 111 }
 112
 113 # removes any git-svn leftovers
 114 cleanup_git_svn_leftovers() {
 115
 116         # Remove any stale git-svn temp files
 117         # The git-svn process creates temp files with random 10 character names
 118         # in the root of $GIT_DIR.  Unfortunately they do not have a recognizable
 119         # prefix, so we just have to kill any files with a 10-character name.
 120         # All characters are chosen from
 121         # [A-Za-z0-9_] so we can at least check that and fortunately the only
 122         # collision is 'FETCH_HEAD' but that doesn't matter.
 123         # There may also be temp files with a Git_ prefix as well.
 124         _randchar='[A-Za-z0-9_]'
 125         _randchar2="$_randchar$_randchar"
 126         _randchar4="$_randchar2$_randchar2"
 127         _randchar10="$_randchar4$_randchar4$_randchar2"
 128         find -L . -maxdepth 1 -type f -name "$_randchar10" -exec rm -f '{}' + || :
 129         find -L . -maxdepth 1 -type f -name "Git_*" -exec rm -f '{}' + || :
 130 }
 131
 132 # removes all leftovers from a previous failed clone attempt
 133 cleanup_failed_clone() {
 134
 135         # Remove any left-over svn-remote.svn or remote.origin config
 136         git config --remove-section svn-remote.svn 2>/dev/null || :
 137         git config --remove-section remote.origin 2>/dev/null || :
 138
 139         # If there is a remote-template.origin section, pre-seed the
 140         # remote.origin section with its contents
 141         git config --get-regexp '^remote-template\.origin\..' |
 142         while read name value; do
 143                 if [ -n "$name" ] && [ -n "$value" ]; then
 144                         git config "remote${name#remote-template}" "$value"
 145                 fi
 146         done
 147
 148         # Any pre-existing FETCH_HEAD from a previous clone failed or not is
 149         # now garbage to be removed
 150         rm -f FETCH_HEAD
 151
 152         # Remove any stale ref locks
 153         clear_stale_ref_locks
 154
 155         # Remove any left-over svn dir from a previous failed attempt
 156         rm -rf svn
 157
 158         # Remove any left-over .darcs dirs from a previous failed attempt
 159         rm -rf *.darcs
 160
 161         # Remove any left-over repo.hg dir from a previous failed attempt
 162         rm -rf repo.hg
 163
 164         # Remove any left-over import/export/temp files from a previous failed attempt
 165         rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks* hg2git-state \
 166                 gfi-marks gfi-packs .pkts-temp .refs-temp
 167
 168         # Remove any git-svn junk
 169         cleanup_git_svn_leftovers
 170
 171         # We want a gc right after the clone, so re-enable that just in case.
 172         # There's a potential race where we could add it and gc.sh could remove
 173         # it, but we'll reunset lastgc just before we remove .delaygc at the end.
 174         [ -e .delaygc ] || >.delaygc
 175         git config --unset gitweb.lastgc 2>/dev/null || :
 176
 177         # Remove all pre-existing refs
 178         rm -f packed-refs
 179         git for-each-ref --format='delete %(refname)' | git_updateref_stdin 2>/dev/null || :
 180
 181         # The initial state before a clone starts has HEAD as a symbolic-ref to master
 182         git symbolic-ref HEAD refs/heads/master
 183
 184         # HEAD is no longer "ok"
 185         git config --unset girocco.headok 2>/dev/null || :
 186
 187         # We, perhaps, ought to remove any packs/loose objects now, but the next gc
 188         # will get rid of any extras.  Also, if we're recloning the same thing, any
 189         # preexisting packs/loose objects containing what we're recloning will only
 190         # speed up the reclone by avoiding some disk writes.  So we don't kill them.
 191
 192         # It's just remotely possible that a bunch of failures in a row could
 193         # create a big mess that just keeps growing and growing...
 194         # Trigger a .needsgc if that happens.
 195         check_and_set_needsgc
 196 }
 197
 198 proj="${1%.git}"
 199 cd "$cfg_reporoot/$proj.git"
 200 bang_reset
 201
 202 ! [ -e .delaygc ] || >.allowgc || :
 203
 204 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 205 echo "Project: $proj"
 206 echo "   Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
 207 echo ""
 208 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 209 url="$(config_get baseurl)" || :
 210 case "$url" in *"       "*|*" "*|"")
 211         echo "Bad mirror URL (\"$url\")"
 212         exit 1
 213 esac
 214
 215 cleanup_failed_clone
 216
 217 # Record original mirror type for use by update.sh
 218 mirror_type="$(get_url_mirror_type "$url")"
 219 git config girocco.mirrortype "$mirror_type"
 220
 221 echo "Mirroring from URL \"$url\""
 222 echo ""
 223
 224 if [ "$cfg_project_owners" = "source" ]; then
 225         config set owner "$(ls -ldH "${url#file://}" 2>/dev/null | LC_ALL=C awk '{print $3}')"
 226 fi
 227
 228 mailaddrs="$(config_get owner)" || :
 229 [ -z "$cfg_admin" ] ||
 230 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 231
 232 # Make sure we don't get any unwanted loose objects
 233 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
 234 # tweak its configuration to prevent that
 235 git_add_config 'fetch.unpackLimit=1'
 236 # Note the git config documentation is wrong
 237 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
 238 git_add_config 'transfer.unpackLimit=1'
 239 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
 240 git_add_config 'fastimport.unpackLimit=0'
 241
 242 # Initial mirror
 243 echo "Initiating mirroring..."
 244 headref=
 245 showheadwarn=
 246 warnempty=
 247
 248 # remember the starting time so we can easily combine fetched loose objects
 249 # we sleep for 1 second after creating .needspack to make sure all objects are newer
 250 if ! [ -e .needspack ]; then
 251         rm -f .needspack
 252         >.needspack
 253         sleep 1
 254 fi
 255
 256 case "$url" in
 257         svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 258                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 259                 # Allow the username to be specified in the "svn-credential.svn.username"
 260                 # property and the password in the "svn-credential.svn.password" property
 261                 # Use an 'anonsvn' username by default as is commonly used for anonymous svn
 262                 # Default the password to the same as the username
 263                 # The password property will be ignored unless a username has been specified
 264                 if svnuser="$(git config --get svn-credential.svn.username)" && [ -n "$svnuser" ]; then
 265                         if ! svnpass="$(git config --get svn-credential.svn.password)"; then
 266                                 svnpass="$svnuser"
 267                         fi
 268                         url1="${url#*://}"
 269                         url1="${url1%%/*}"
 270                         case "$url1" in ?*"@"?*)
 271                                 urlsch="${url%%://*}"
 272                                 url="$urlsch://${url#*@}"
 273                         esac
 274                 else
 275                         # As a fallback, check in the URL, just in case
 276                         url1="${url#*://}"
 277                         url1="${url1%%/*}"
 278                         svnuser=
 279                         case "$url1" in ?*"@"?*)
 280                                 urlsch="${url%%://*}"
 281                                 url="$urlsch://${url#*@}"
 282                                 url1="${url1%%@*}"
 283                                 svnuser="${url1%%:*}"
 284                                 if [ -n "$svnuser" ]; then
 285                                         svnpass="$svnuser"
 286                                         case "$url1" in *":"*)
 287                                                 svnpass="${url1#*:}"
 288                                         esac
 289                                 fi
 290                         esac
 291                         if [ -z "$svnuser" ]; then
 292                                 svnuser="anonsvn"
 293                                 svnpass="anonsvn"
 294                         fi
 295                 fi
 296                 GIT_ASKPASS_PASSWORD="$svnpass"
 297                 export GIT_ASKPASS_PASSWORD
 298                 # We just remove svn+ here, so svn+http://... becomes http://...
 299                 # We also remove a trailing '/' to match what git-svn will do
 300                 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
 301                 svnurl="${svnurl%/}"
 302                 # We require svn info to succeed on the URL otherwise it's
 303                 # simply not a valid URL and without using -s on the init it
 304                 # will not otherwise be tested until the fetch
 305                 svn --non-interactive --username "$svnuser" --password "$svnpass" info "$svnurl" >/dev/null
 306                 # We initially use -s for the init which will possibly shorten
 307                 # the URL.  However, the shortening can fail if a password is
 308                 # not required for the longer version but is for the shorter,
 309                 # so try again without -s if the -s version fails.
 310                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 311                 # create a .git subdirectory!
 312                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" -s "$svnurl" <"$mtlinesfile" ||
 313                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" "$svnurl" <"$mtlinesfile"
 314                 # We need to remember this url so we can detect changes because
 315                 # ever so "helpful" git-svn may shorten it!
 316                 config_set svnurl "$svnurl"
 317                 # At this point, since we asked for a standard layout (-s) git-svn
 318                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 319                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 320                 # config items.  We could avoid this by not using the '-s' option
 321                 # but then we might not get all the history.  If, for example, we
 322                 # are cloning an http://svn.example.com/repos/public repository that
 323                 # early in its history moved trunk => public/trunk we would miss that
 324                 # earlier history without allowing the funky shorten+prefix behavior.
 325                 # So we read back the svn-remote.svn.fetch configuration and compute
 326                 # the prefix.  This way we are sure to get the correct prefix.
 327                 gitsvnurl="$(git config --get svn-remote.svn.url)" || :
 328                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1)" || :
 329                 gitsvnprefix="${gitsvnfetch%%:*}"
 330                 gitsvnsuffix="${gitsvnprefix##*/}"
 331                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 332                 # Ask git-svn to store everything in the normal non-remote
 333                 # locations being careful to use the correct prefix
 334                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 335                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 336                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 337                 # look for additional non-standard directories to fetch
 338                 # check for standard layout at the same time
 339                 foundstd=
 340                 foundfile=
 341                 svn --non-interactive --username "$svnuser" --password "$svnpass" ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null |
 342                 { while read file; do case $file in
 343                         # skip the already-handled standard ones and any with a space or tab
 344                         *'      '*|*' '*) :;;
 345                         trunk/|branches/|tags/) foundstd=1;;
 346                         # only fetch extra directories from the $svnurl root (not any files)
 347                         *?/) git config --add svn-remote.svn.fetch \
 348                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 349                         *?) foundfile=1;;
 350                 esac; done
 351                 # if files found and no standard directories present use a simpler layout
 352                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 353                         git config --unset svn-remote.svn.branches
 354                         git config --unset svn-remote.svn.tags
 355                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 356                 fi; }
 357                 test $? -eq 0
 358                 # git svn fetch on a very large repo can take some time and the
 359                 # remote server may interrupt the connection from time to time.
 360                 # keep retrying (after a brief pause) as long as we are making progress.
 361                 # however, we do limit the total number of retries to 1000
 362                 # we will, however, retry up to 5 times even if we're not making progress
 363                 v_get_svn_progress_fingerprint() {
 364                         eval "$1="'"$({ GIT_DIR=. git svn info <"$mtlinesfile" 2>&1; git show-ref --head 2>&1; } |
 365                         git hash-object -t blob --stdin )"' || :
 366                 }
 367                 svn_ret_err() { return "${1:-1}"; }
 368                 svn_retries=1000 # maximum possible fetch attempts no matter what
 369                 svn_retry_backoff_start_half=60 # min retry wait is double this amount in seconds
 370                 svn_backoff_count=7 # max retry wait is $svn_retry_backoff_start_half * 2^$svn_backoff_count
 371                 # Cumulative backoff wait before giving up on consecutive no-progress retries
 372                 #   is approximately 2 * $svn_retry_backoff_start_half * 2^$svn_backoff_count
 373                 # For a $svn_backoff_count of 7 that works out to be exactly 4h14m
 374                 svn_progress=
 375                 v_get_svn_progress_fingerprint svn_progress
 376                 svn_progress_retries="$svn_retries"
 377                 svn_retry_backoff="$svn_retry_backoff_start_half"
 378                 svn_err=0
 379                 while [ "$svn_retries" -gt 0 ]; do
 380                         svn_retries="$(( $svn_retries - 1 ))"
 381                         svn_err=0
 382                         GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1
 383                         export GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 384                         unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 385                         saveconfig="$GIT_CONFIG_PARAMETERS"
 386                         git_add_config 'gc.auto=1'
 387                         git_add_config 'gc.autoPackLimit=1'
 388                         # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 389                         GIT_DIR=. git svn fetch --log-window-size=$var_log_window_size --username="$svnuser" --quiet <"$mtlinesfile" || svn_err="$?"
 390                         GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 391                         export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 392                         unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 393                         unset GIT_CONFIG_PARAMETERS
 394                         [ -z "$saveconfig" ] || {
 395                                 GIT_CONFIG_PARAMETERS="$saveconfig"
 396                                 export GIT_CONFIG_PARAMETERS
 397                         }
 398                         [ "${svn_err:-1}" -ne 0 ] || break # success!
 399                         # Check to see if we made any progress
 400                         v_get_svn_progress_fingerprint svn_progress_now
 401                         if [ "$svn_progress_now" != "$svn_progress" ]; then
 402                                 # we made progress, continue the loop with min wait
 403                                 svn_progress="$svn_progress_now"
 404                                 svn_progress_retries="$svn_retries"
 405                                 svn_retry_backoff="$svn_retry_backoff_start_half"
 406                         else
 407                                 # no progress, but we only give up after
 408                                 # $svn_backoff_count no-progress attempts in a row
 409                                 [ "$(( $svn_progress_retries - $svn_retries ))" -lt "$svn_backoff_count" ] ||
 410                                         break # failure
 411                                 # continue but only after twice the previous wait
 412                                 # (which will still be the min wait if this is the
 413                                 # first no-progress retry after making some progress)
 414                         fi
 415                         svn_retry_backoff="$(( 2 * $svn_retry_backoff ))"
 416                         # Pause for $svn_retry_backoff seconds before retrying to be friendly to the server
 417                         # Use that time to pack up loose objects if there are "lotsa" them
 418                         if ! lotsa_loose_objects_or_sopacks; then
 419                                 echo "Pausing for $svn_retry_backoff seconds before retrying ($(date))"
 420                                 sleep "$svn_retry_backoff"
 421                         else
 422                                 pausestop="$(( $(date '+%s') + $svn_retry_backoff ))"
 423                                 echo "Pausing and packing loose objects for $svn_retry_backoff seconds before retrying ($(date))"
 424                                 pack_incremental_loose_objects_if_lockable ||
 425                                 echo "Packing skipped (only pausing): $lockerr"
 426                                 timenow="$(date '+%s')"
 427                                 if [ "$timenow" -lt "$pausestop" ]; then
 428                                         sleepamt="$(( $pausestop - $timenow ))"
 429                                         [ "$sleepamt" -le "$svn_retry_backoff" ] ||
 430                                                 sleepamt="$svn_retry_backoff" # paranoia check
 431                                         sleep "$sleepamt"
 432                                 fi
 433                         fi
 434                         cleanup_git_svn_leftovers
 435                         echo "Retrying fetch ($(date))"
 436                 done
 437                 [ "${svn_err:-1}" -eq 0 ] || svn_ret_err "$svn_err"
 438                 test ${svn_err:-1} -eq 0
 439                 # git svn does not preserve group permissions in the svn subdirectory
 440                 chmod -R ug+rw,o+r svn
 441                 # git svn also leaves behind ref turds that end with @nnn
 442                 # We get rid of them now
 443                 git for-each-ref --format='%(refname)' |
 444                 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
 445                 git_updateref_stdin
 446                 unset GIT_ASKPASS_PASSWORD
 447                 ;;
 448         darcs://* | darcs+http://* | darcs+https://*)
 449                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 450                 case "$url" in
 451                         darcs://*) darcsurl="http://${url#darcs://}";;
 452                         *) darcsurl="${url#darcs+}";;
 453                 esac
 454                 git_darcs_fetch "$darcsurl"
 455                 ;;
 456         bzr://*)
 457                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 458                 # we just remove bzr:// here, a typical bzr url is just
 459                 # "lp:foo"
 460                 bzrurl="${url#bzr://}"
 461                 git_bzr_fetch "$bzrurl"
 462                 ;;
 463         hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
 464                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 465                 # We just remove hg+ here, so hg+http://... becomes http://...
 466                 hgurl="${url#hg+}"
 467                 # Perform the initial hg clone
 468                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 469                 # Do the fast-export | fast-import
 470                 git_hg_fetch
 471                 ;;
 472         *)
 473                 # We manually add remote.origin.url and remote.origin.fetch
 474                 # to simulate a `git remote add --mirror=fetch` since that's
 475                 # not available until Git 1.7.5 and this way we guarantee we
 476                 # always get exactly the intended configuration and nothing else.
 477                 git config remote.origin.url "$url"
 478                 if ! is_gfi_mirror_url "$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
 479                         git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
 480                         git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
 481                         git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
 482                         git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
 483                         git config --bool girocco.lastupdateclean true
 484                 else
 485                         git config --replace-all remote.origin.fetch "+refs/*:refs/*"
 486                         git config --bool girocco.lastupdateclean false
 487                 fi
 488                 # Set the correct HEAD symref by using ls-remote first
 489                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp ||
 490                 {
 491                         # Since everything was redirected, on failure there'd be no output,
 492                         # so let's make some failure output
 493                         cat .pkts-temp
 494                         echo ""
 495                         echo "git ls-remote \"$url\" failed"
 496                         exit 1
 497                 }
 498                 # Compensate for git() {} side effects
 499                 unset GIT_TRACE_PACKET
 500                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 501                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 502                 # out of the packet trace data.
 503                 if [ -s .refs-temp ]; then
 504                         # Nothing to do unless the remote repository has at least 1 ref
 505                         # See if we got a HEAD ref
 506                         head="$(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+HEAD\$" <.refs-temp | LC_ALL=C awk '{print $1}')"
 507                         # If the remote has HEAD set to a symbolic ref that does not exist
 508                         # then we will not receive a HEAD ref in the ls-remote output
 509                         headref=
 510                         showheadwarn=
 511                         symrefcap=
 512                         if [ -n "$head" ]; then
 513                                 symrefcap="$(LC_ALL=C sed -ne <.pkts-temp \
 514                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 515                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 516                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 517                                 # matches HEAD otherwise take the first refs/heads/... match
 518                                 matchcnt=0
 519                                 while read ref; do
 520                                         [ -n "$ref" ] || continue
 521                                         matchcnt=$(( $matchcnt + 1 ))
 522                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 523                                                 headref="$ref"
 524                                         fi
 525                                         if [ "$headref" = "${symrefcap:-refs/heads/master}" ] && [ $matchcnt -gt 1 ]; then
 526                                                 break
 527                                         fi
 528                                 done <<-EOT
 529                                 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 530                                         LC_ALL=C awk '{print $2}')
 531                                 EOT
 532                                 # Warn if there was more than one match and $symrefcap is empty
 533                                 # or $symrefcap is not the same as $headref since our choice might
 534                                 # differ from the source repository's HEAD
 535                                 if [ $matchcnt -ge 1 ] && [ "$symrefcap" != "$headref" ] &&
 536                                    { [ -n "$symrefcap" ] || [ $matchcnt -gt 1 ]; }; then
 537                                         showheadwarn=1
 538                                 fi
 539                         fi
 540                         if [ -z "$headref" ]; then
 541                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 542                                 # if it exists otherwise take the first refs/heads/...
 543                                 # We do not support having a detached HEAD.
 544                                 # We always warn now because we will be setting HEAD differently
 545                                 # than the source repository had HEAD set
 546                                 showheadwarn=1
 547                                 while read ref; do
 548                                         [ -n "$ref" ] || continue
 549                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 550                                                 headref="$ref"
 551                                         fi
 552                                         [ "$headref" != "refs/heads/master" ] || break
 553                                 done <<-EOT
 554                                 $(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 555                                         LC_ALL=C awk '{print $2}')
 556                                 EOT
 557                         fi
 558                         # If we STILL do not have a HEAD ref (perhaps the source repository
 559                         # contains only tags) then use refs/heads/master.  It will be invalid
 560                         # but is no worse than we used to do by default and we'll warn about
 561                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 562                         [ -n "$headref" ] || headref="refs/heads/master"
 563                         git symbolic-ref HEAD "$headref"
 564                         pruneopt=--prune
 565                         [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
 566                         # remember the starting time so we can easily detect new packs for fast-import mirrors
 567                         # we sleep for 1 second after creating .gfipack to make sure all packs are newer
 568                         if is_gfi_mirror_url "$url" && ! [ -e .gfipack ]; then
 569                                 rm -f .gfipack
 570                                 >.gfipack
 571                                 sleep 1
 572                         fi
 573                         GIT_SSL_NO_VERIFY=1 git remote update $pruneopt
 574                         if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
 575                                 find -L objects/pack -type f -newer .gfipack -name "pack-$octet20*.pack" -print >>gfi-packs
 576                                 rm -f .gfipack
 577                         fi
 578                 else
 579                         warnempty=1
 580                         git symbolic-ref HEAD "refs/heads/master"
 581                 fi
 582                 rm -f .refs-temp .pkts-temp
 583                 ;;
 584 esac
 585
 586 # The objects subdirectories permissions must be updated now.
 587 # In the case of a dumb http clone, the permissions will not be correct
 588 # (missing group write) despite the core.sharedrepository=1 setting!
 589 # The objects themselves seem to have the correct permissions.
 590 # This problem appears to have been fixed in the most recent git versions.
 591 perms=g+w
 592 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 593 chmod $perms $(find -L objects -maxdepth 1 -type d) 2>/dev/null || :
 594
 595 # We may have just cloned a lot of refs and they will all be
 596 # individual files at this point.  Let's pack them now so we
 597 # can have better performance right from the start.
 598 git pack-refs --all
 599
 600 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 601 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 602 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 603 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 604         --count=1 refs/heads >info/lastactivity || :
 605 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 606
 607 # Don't leave a multi-megabyte useless FETCH_HEAD behind
 608 rm -f FETCH_HEAD
 609
 610 # Last ditch attempt to get a valid HEAD for a non-git source
 611 check_and_set_head || :
 612
 613 # The rest
 614 echo "Final touches..."
 615 git update-server-info
 616 trap "" EXIT
 617
 618 # run gc now unless the clone is empty
 619 if [ -z "$warnempty" ]; then
 620         git config --unset gitweb.lastgc 2>/dev/null || :
 621         rm -f .delaygc .allowgc
 622 fi
 623
 624 emptynote=
 625 [ -z "$warnempty" ] ||
 626 emptynote="
 627 WARNING:  You have mirrored an empty repository.
 628 "
 629 headnote=
 630 [ -z "$showheadwarn" ] || [ -z "$headref" ] ||
 631 headnote="
 632 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 633        Use the \"Project settings\" link to choose a different HEAD symref.
 634 "
 635 sizenote=
 636 ! is_gfi_mirror ||
 637 sizenote="
 638 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 639        size may be somewhat larger than necessary.  This will be corrected
 640        shortly.  If you intend to clone this repository you may want to
 641        wait up to 1 hour before doing so in order to receive the more
 642        compact final size.
 643 "
 644 [ -z "$mailaddrs" ] ||
 645 mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 646 Congratulations! The clone of project $proj just completed.
 647
 648         * Source URL: $url
 649         * GitWeb interface: $cfg_gitweburl/$proj.git
 650         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 651 $emptynote$headnote$sizenote
 652 Have a lot of fun.
 653 EOT
 654
 655 echo "Mirroring finished successfuly!"
 656 # In case this is a re-mirror, lastgc could have been set already so clear it now
 657 git config --unset gitweb.lastgc || :
 658 rm .clone_in_progress
 659 echo "$sizenote@OVER@"