taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6 . @basedir@/jobd/gc-util-functions.sh
   7
   8 set -e
   9
  10 umask 002
  11 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  12 clean_git_env
  13
  14 # darcs fast-export | git fast-import with error handling
  15 git_darcs_fetch() (
  16         set_utf8_locale
  17         _err1=
  18         _err2=
  19         exec 3>&1
  20         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  21         $(
  22                 exec 4>&3 3>&1 1>&4 4>&-
  23                 {
  24                         _e1=0
  25                         "$cfg_basedir"/bin/darcs-fast-export \
  26                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  27                         echo $_e1 >&3
  28                 } |
  29                 {
  30                         _e2=0
  31                         git fast-import \
  32                                 --export-marks="$(pwd)/gfi-marks" \
  33                                 --export-pack-edges="$(pwd)/gfi-packs" \
  34                                 --force 3>&- || _e2=$?
  35                         echo $_e2 >&3
  36                 }
  37         )
  38         EOT
  39         exec 3>&-
  40         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  41         return $?
  42 )
  43
  44 # bzr fast-export | git fast-import with error handling
  45 git_bzr_fetch() (
  46         set_utf8_locale
  47         BZR_LOG=/dev/null
  48         export BZR_LOG
  49         _err1=
  50         _err2=
  51         exec 3>&1
  52         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  53         $(
  54                 exec 4>&3 3>&1 1>&4 4>&-
  55                 {
  56                         _e1=0
  57                         bzr fast-export --plain \
  58                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  59                         echo $_e1 >&3
  60                 } |
  61                 {
  62                         _e2=0
  63                         git fast-import \
  64                                 --export-marks="$(pwd)/gfi-marks" \
  65                                 --export-pack-edges="$(pwd)/gfi-packs" \
  66                                 --force 3>&- || _e2=$?
  67                         echo $_e2 >&3
  68                 }
  69         )
  70         EOT
  71         exec 3>&-
  72         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  73         return $?
  74 )
  75
  76 send_clone_failed() {
  77         trap "" EXIT
  78         # We must now close the .clonelog file that is open on stdout and stderr
  79         exec >/dev/null 2>&1
  80         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
  81         failaddrs="$(config_get owner)" || :
  82         [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
  83         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
  84         [ -z "$failaddrs" ] ||
  85         {
  86               cat <<EOT
  87 Condolences. The clone of project $proj just failed.
  88
  89         * Source URL: $url
  90         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
  91
  92 The project settings link may be used to adjust the settings
  93 and restart the clone in order to try the clone again.
  94 EOT
  95                 if [ -f .clonelog ] && [ -r .clonelog ]; then
  96                         echo ""
  97                         echo "Log follows:"
  98                         echo ""
  99                         loglines=$(LC_ALL=C wc -l <.clonelog)
 100                         if [ $loglines -le 203 ]; then
 101                                 cat .clonelog
 102                         else
 103                                 head -n 100 .clonelog
 104                                 echo ""
 105                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
 106                                 echo ""
 107                                 tail -n 100 .clonelog
 108                         fi
 109                 fi
 110         } | mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 111 }
 112
 113 # removes any git-svn leftovers
 114 cleanup_git_svn_leftovers() {
 115
 116         # Remove any stale git-svn temp files
 117         # The git-svn process creates temp files with random 10 character names
 118         # in the root of $GIT_DIR.  Unfortunately they do not have a recognizable
 119         # prefix, so we just have to kill any files with a 10-character name.
 120         # All characters are chosen from
 121         # [A-Za-z0-9_] so we can at least check that and fortunately the only
 122         # collision is 'FETCH_HEAD' but that doesn't matter.
 123         # There may also be temp files with a Git_ prefix as well.
 124         _randchar='[A-Za-z0-9_]'
 125         _randchar2="$_randchar$_randchar"
 126         _randchar4="$_randchar2$_randchar2"
 127         _randchar10="$_randchar4$_randchar4$_randchar2"
 128         find -L . -maxdepth 1 -type f -name "$_randchar10" -exec rm -f '{}' + || :
 129         find -L . -maxdepth 1 -type f -name "Git_*" -exec rm -f '{}' + || :
 130 }
 131
 132 # removes all leftovers from a previous failed clone attempt
 133 cleanup_failed_clone() {
 134
 135         # Remove any left-over svn-remote.svn or remote.origin config
 136         git config --remove-section svn-remote.svn 2>/dev/null || :
 137         git config --remove-section remote.origin 2>/dev/null || :
 138
 139         # If there is a remote-template.origin section, pre-seed the
 140         # remote.origin section with its contents
 141         git config --get-regexp '^remote-template\.origin\..' |
 142         while read name value; do
 143                 if [ -n "$name" ] && [ -n "$value" ]; then
 144                         git config "remote${name#remote-template}" "$value"
 145                 fi
 146         done
 147
 148         # Any pre-existing FETCH_HEAD from a previous clone failed or not is
 149         # now garbage to be removed
 150         rm -f FETCH_HEAD
 151
 152         # Remove any left-over svn dir from a previous failed attempt
 153         rm -rf svn
 154
 155         # Remove any left-over .darcs dirs from a previous failed attempt
 156         rm -rf *.darcs
 157
 158         # Remove any left-over repo.hg dir from a previous failed attempt
 159         rm -rf repo.hg
 160
 161         # Remove any left-over import/export/temp files from a previous failed attempt
 162         rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks* hg2git-state \
 163                 gfi-marks gfi-packs .pkts-temp .refs-temp
 164
 165         # Remove any git-svn junk
 166         cleanup_git_svn_leftovers
 167
 168         # We want a gc right after the clone, so re-enable that just in case.
 169         # There's a potential race where we could add it and gc.sh could remove
 170         # it, but we'll reunset lastgc just before we remove .delaygc at the end.
 171         [ -e .delaygc ] || >.delaygc
 172         git config --unset gitweb.lastgc 2>/dev/null || :
 173
 174         # Remove all pre-existing refs
 175         rm -f packed-refs
 176         git for-each-ref --format='delete %(refname)' | git_updateref_stdin 2>/dev/null || :
 177
 178         # The initial state before a clone starts has HEAD as a symbolic-ref to master
 179         git symbolic-ref HEAD refs/heads/master
 180
 181         # HEAD is no longer "ok"
 182         git config --unset girocco.headok 2>/dev/null || :
 183
 184         # We, perhaps, ought to remove any packs/loose objects now, but the next gc
 185         # will get rid of any extras.  Also, if we're recloning the same thing, any
 186         # preexisting packs/loose objects containing what we're recloning will only
 187         # speed up the reclone by avoiding some disk writes.  So we don't kill them.
 188
 189         # It's just remotely possible that a bunch of failures in a row could
 190         # create a big mess that just keeps growing and growing...
 191         # Trigger a .needsgc if that happens.
 192         check_and_set_needsgc
 193 }
 194
 195 proj="${1%.git}"
 196 cd "$cfg_reporoot/$proj.git"
 197 bang_reset
 198
 199 ! [ -e .delaygc ] || >.allowgc || :
 200
 201 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 202 echo "Project: $proj"
 203 echo "   Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
 204 echo ""
 205 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 206 url="$(config_get baseurl)" || :
 207 case "$url" in *"       "*|*" "*|"")
 208         echo "Bad mirror URL (\"$url\")"
 209         exit 1
 210 esac
 211
 212 cleanup_failed_clone
 213
 214 # Record original mirror type for use by update.sh
 215 mirror_type="$(get_url_mirror_type "$url")"
 216 git config girocco.mirrortype "$mirror_type"
 217
 218 echo "Mirroring from URL \"$url\""
 219 echo ""
 220
 221 if [ "$cfg_project_owners" = "source" ]; then
 222         config set owner "$(ls -ldH "${url#file://}" 2>/dev/null | LC_ALL=C awk '{print $3}')"
 223 fi
 224
 225 mailaddrs="$(config_get owner)" || :
 226 [ -z "$cfg_admin" ] ||
 227 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 228
 229 # Make sure we don't get any unwanted loose objects
 230 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
 231 # tweak its configuration to prevent that
 232 git_add_config 'fetch.unpackLimit=1'
 233 # Note the git config documentation is wrong
 234 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
 235 git_add_config 'transfer.unpackLimit=1'
 236 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
 237 git_add_config 'fastimport.unpackLimit=0'
 238
 239 # Initial mirror
 240 echo "Initiating mirroring..."
 241 headref=
 242 showheadwarn=
 243 warnempty=
 244
 245 # remember the starting time so we can easily combine fetched loose objects
 246 # we sleep for 1 second after creating .needspack to make sure all objects are newer
 247 if ! [ -e .needspack ]; then
 248         rm -f .needspack
 249         >.needspack
 250         sleep 1
 251 fi
 252
 253 case "$url" in
 254         svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 255                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 256                 # Allow the username to be specified in the "svn-credential.svn.username"
 257                 # property and the password in the "svn-credential.svn.password" property
 258                 # Use an 'anonsvn' username by default as is commonly used for anonymous svn
 259                 # Default the password to the same as the username
 260                 # The password property will be ignored unless a username has been specified
 261                 if svnuser="$(git config --get svn-credential.svn.username)" && [ -n "$svnuser" ]; then
 262                         if ! svnpass="$(git config --get svn-credential.svn.password)"; then
 263                                 svnpass="$svnuser"
 264                         fi
 265                         url1="${url#*://}"
 266                         url1="${url1%%/*}"
 267                         case "$url1" in ?*"@"?*)
 268                                 urlsch="${url%%://*}"
 269                                 url="$urlsch://${url#*@}"
 270                         esac
 271                 else
 272                         # As a fallback, check in the URL, just in case
 273                         url1="${url#*://}"
 274                         url1="${url1%%/*}"
 275                         svnuser=
 276                         case "$url1" in ?*"@"?*)
 277                                 urlsch="${url%%://*}"
 278                                 url="$urlsch://${url#*@}"
 279                                 url1="${url1%%@*}"
 280                                 svnuser="${url1%%:*}"
 281                                 if [ -n "$svnuser" ]; then
 282                                         svnpass="$svnuser"
 283                                         case "$url1" in *":"*)
 284                                                 svnpass="${url1#*:}"
 285                                         esac
 286                                 fi
 287                         esac
 288                         if [ -z "$svnuser" ]; then
 289                                 svnuser="anonsvn"
 290                                 svnpass="anonsvn"
 291                         fi
 292                 fi
 293                 GIT_ASKPASS_PASSWORD="$svnpass"
 294                 export GIT_ASKPASS_PASSWORD
 295                 # We just remove svn+ here, so svn+http://... becomes http://...
 296                 # We also remove a trailing '/' to match what git-svn will do
 297                 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
 298                 svnurl="${svnurl%/}"
 299                 # We require svn info to succeed on the URL otherwise it's
 300                 # simply not a valid URL and without using -s on the init it
 301                 # will not otherwise be tested until the fetch
 302                 svn --non-interactive --username "$svnuser" --password "$svnpass" info "$svnurl" >/dev/null
 303                 # We initially use -s for the init which will possibly shorten
 304                 # the URL.  However, the shortening can fail if a password is
 305                 # not required for the longer version but is for the shorter,
 306                 # so try again without -s if the -s version fails.
 307                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 308                 # create a .git subdirectory!
 309                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" -s "$svnurl" <"$mtlinesfile" ||
 310                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" "$svnurl" <"$mtlinesfile"
 311                 # We need to remember this url so we can detect changes because
 312                 # ever so "helpful" git-svn may shorten it!
 313                 config_set svnurl "$svnurl"
 314                 # At this point, since we asked for a standard layout (-s) git-svn
 315                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 316                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 317                 # config items.  We could avoid this by not using the '-s' option
 318                 # but then we might not get all the history.  If, for example, we
 319                 # are cloning an http://svn.example.com/repos/public repository that
 320                 # early in its history moved trunk => public/trunk we would miss that
 321                 # earlier history without allowing the funky shorten+prefix behavior.
 322                 # So we read back the svn-remote.svn.fetch configuration and compute
 323                 # the prefix.  This way we are sure to get the correct prefix.
 324                 gitsvnurl="$(git config --get svn-remote.svn.url)" || :
 325                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1)" || :
 326                 gitsvnprefix="${gitsvnfetch%%:*}"
 327                 gitsvnsuffix="${gitsvnprefix##*/}"
 328                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 329                 # Ask git-svn to store everything in the normal non-remote
 330                 # locations being careful to use the correct prefix
 331                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 332                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 333                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 334                 # look for additional non-standard directories to fetch
 335                 # check for standard layout at the same time
 336                 foundstd=
 337                 foundfile=
 338                 svn --non-interactive --username "$svnuser" --password "$svnpass" ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null |
 339                 { while read file; do case $file in
 340                         # skip the already-handled standard ones and any with a space or tab
 341                         *'      '*|*' '*) :;;
 342                         trunk/|branches/|tags/) foundstd=1;;
 343                         # only fetch extra directories from the $svnurl root (not any files)
 344                         *?/) git config --add svn-remote.svn.fetch \
 345                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 346                         *?) foundfile=1;;
 347                 esac; done
 348                 # if files found and no standard directories present use a simpler layout
 349                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 350                         git config --unset svn-remote.svn.branches
 351                         git config --unset svn-remote.svn.tags
 352                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 353                 fi; }
 354                 test $? -eq 0
 355                 # git svn fetch on a very large repo can take some time and the
 356                 # remote server may interrupt the connection from time to time.
 357                 # keep retrying (after a brief pause) as long as we are making progress.
 358                 # however, we do limit the total number of retries to 1000
 359                 # we will, however, retry up to 5 times even if we're not making progress
 360                 v_get_svn_progress_fingerprint() {
 361                         eval "$1="'"$({ GIT_DIR=. git svn info <"$mtlinesfile" 2>&1; git show-ref --head 2>&1; } |
 362                         git hash-object -t blob --stdin )"' || :
 363                 }
 364                 svn_ret_err() { return "${1:-1}"; }
 365                 svn_retries=1000 # maximum possible fetch attempts no matter what
 366                 svn_retry_backoff_start_half=60 # min retry wait is double this amount in seconds
 367                 svn_backoff_count=7 # max retry wait is $svn_retry_backoff_start_half * 2^$svn_backoff_count
 368                 # Cumulative backoff wait before giving up on consecutive no-progress retries
 369                 #   is approximately 2 * $svn_retry_backoff_start_half * 2^$svn_backoff_count
 370                 # For a $svn_backoff_count of 7 that works out to be exactly 4h14m
 371                 svn_progress=
 372                 v_get_svn_progress_fingerprint svn_progress
 373                 svn_progress_retries="$svn_retries"
 374                 svn_retry_backoff="$svn_retry_backoff_start_half"
 375                 svn_err=0
 376                 while [ "$svn_retries" -gt 0 ]; do
 377                         svn_retries="$(( $svn_retries - 1 ))"
 378                         svn_err=0
 379                         GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1
 380                         export GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 381                         unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 382                         saveconfig="$GIT_CONFIG_PARAMETERS"
 383                         git_add_config 'gc.auto=1'
 384                         git_add_config 'gc.autoPackLimit=1'
 385                         # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 386                         GIT_DIR=. git svn fetch --log-window-size=$var_log_window_size --username="$svnuser" --quiet <"$mtlinesfile" || svn_err="$?"
 387                         GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 388                         export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 389                         unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 390                         unset GIT_CONFIG_PARAMETERS
 391                         [ -z "$saveconfig" ] || {
 392                                 GIT_CONFIG_PARAMETERS="$saveconfig"
 393                                 export GIT_CONFIG_PARAMETERS
 394                         }
 395                         [ "${svn_err:-1}" -ne 0 ] || break # success!
 396                         # Check to see if we made any progress
 397                         v_get_svn_progress_fingerprint svn_progress_now
 398                         if [ "$svn_progress_now" != "$svn_progress" ]; then
 399                                 # we made progress, continue the loop with min wait
 400                                 svn_progress="$svn_progress_now"
 401                                 svn_progress_retries="$svn_retries"
 402                                 svn_retry_backoff="$svn_retry_backoff_start_half"
 403                         else
 404                                 # no progress, but we only give up after
 405                                 # $svn_backoff_count no-progress attempts in a row
 406                                 [ "$(( $svn_progress_retries - $svn_retries ))" -lt "$svn_backoff_count" ] ||
 407                                         break # failure
 408                                 # continue but only after twice the previous wait
 409                                 # (which will still be the min wait if this is the
 410                                 # first no-progress retry after making some progress)
 411                         fi
 412                         svn_retry_backoff="$(( 2 * $svn_retry_backoff ))"
 413                         # Pause for $svn_retry_backoff seconds before retrying to be friendly to the server
 414                         # Use that time to pack up loose objects if there are "lotsa" them
 415                         if ! lotsa_loose_objects_or_sopacks; then
 416                                 echo "Pausing for $svn_retry_backoff seconds before retrying ($(date))"
 417                                 sleep "$svn_retry_backoff"
 418                         else
 419                                 pausestop="$(( $(date '+%s') + $svn_retry_backoff ))"
 420                                 echo "Pausing and packing loose objects for $svn_retry_backoff seconds before retrying ($(date))"
 421                                 pack_incremental_loose_objects_if_lockable ||
 422                                 echo "Packing skipped (only pausing): $lockerr"
 423                                 timenow="$(date '+%s')"
 424                                 if [ "$timenow" -lt "$pausestop" ]; then
 425                                         sleepamt="$(( $pausestop - $timenow ))"
 426                                         [ "$sleepamt" -le "$svn_retry_backoff" ] ||
 427                                                 sleepamt="$svn_retry_backoff" # paranoia check
 428                                         sleep "$sleepamt"
 429                                 fi
 430                         fi
 431                         cleanup_git_svn_leftovers
 432                         echo "Retrying fetch ($(date))"
 433                 done
 434                 [ "${svn_err:-1}" -eq 0 ] || svn_ret_err "$svn_err"
 435                 test ${svn_err:-1} -eq 0
 436                 # git svn does not preserve group permissions in the svn subdirectory
 437                 chmod -R ug+rw,o+r svn
 438                 # git svn also leaves behind ref turds that end with @nnn
 439                 # We get rid of them now
 440                 git for-each-ref --format='%(refname)' |
 441                 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
 442                 git_updateref_stdin
 443                 unset GIT_ASKPASS_PASSWORD
 444                 ;;
 445         darcs://* | darcs+http://* | darcs+https://*)
 446                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 447                 case "$url" in
 448                         darcs://*) darcsurl="http://${url#darcs://}";;
 449                         *) darcsurl="${url#darcs+}";;
 450                 esac
 451                 git_darcs_fetch "$darcsurl"
 452                 ;;
 453         bzr://*)
 454                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 455                 # we just remove bzr:// here, a typical bzr url is just
 456                 # "lp:foo"
 457                 bzrurl="${url#bzr://}"
 458                 git_bzr_fetch "$bzrurl"
 459                 ;;
 460         hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
 461                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 462                 # We just remove hg+ here, so hg+http://... becomes http://...
 463                 hgurl="${url#hg+}"
 464                 # Perform the initial hg clone
 465                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 466                 # Do the fast-export | fast-import
 467                 git_hg_fetch
 468                 ;;
 469         *)
 470                 # We manually add remote.origin.url and remote.origin.fetch
 471                 # to simulate a `git remote add --mirror=fetch` since that's
 472                 # not available until Git 1.7.5 and this way we guarantee we
 473                 # always get exactly the intended configuration and nothing else.
 474                 git config remote.origin.url "$url"
 475                 if ! is_gfi_mirror_url "$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
 476                         git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
 477                         git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
 478                         git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
 479                         git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
 480                         git config --bool girocco.lastupdateclean true
 481                 else
 482                         git config --replace-all remote.origin.fetch "+refs/*:refs/*"
 483                         git config --bool girocco.lastupdateclean false
 484                 fi
 485                 # Set the correct HEAD symref by using ls-remote first
 486                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp ||
 487                 {
 488                         # Since everything was redirected, on failure there'd be no output,
 489                         # so let's make some failure output
 490                         cat .pkts-temp
 491                         echo ""
 492                         echo "git ls-remote \"$url\" failed"
 493                         exit 1
 494                 }
 495                 # Compensate for git() {} side effects
 496                 unset GIT_TRACE_PACKET
 497                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 498                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 499                 # out of the packet trace data.
 500                 if [ -s .refs-temp ]; then
 501                         # Nothing to do unless the remote repository has at least 1 ref
 502                         # See if we got a HEAD ref
 503                         head="$(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+HEAD\$" <.refs-temp | LC_ALL=C awk '{print $1}')"
 504                         # If the remote has HEAD set to a symbolic ref that does not exist
 505                         # then we will not receive a HEAD ref in the ls-remote output
 506                         headref=
 507                         showheadwarn=
 508                         symrefcap=
 509                         if [ -n "$head" ]; then
 510                                 symrefcap="$(LC_ALL=C sed -ne <.pkts-temp \
 511                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 512                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 513                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 514                                 # matches HEAD otherwise take the first refs/heads/... match
 515                                 matchcnt=0
 516                                 while read ref; do
 517                                         [ -n "$ref" ] || continue
 518                                         matchcnt=$(( $matchcnt + 1 ))
 519                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 520                                                 headref="$ref"
 521                                         fi
 522                                         if [ "$headref" = "${symrefcap:-refs/heads/master}" ] && [ $matchcnt -gt 1 ]; then
 523                                                 break
 524                                         fi
 525                                 done <<-EOT
 526                                 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 527                                         LC_ALL=C awk '{print $2}')
 528                                 EOT
 529                                 # Warn if there was more than one match and $symrefcap is empty
 530                                 # or $symrefcap is not the same as $headref since our choice might
 531                                 # differ from the source repository's HEAD
 532                                 if [ $matchcnt -ge 1 ] && [ "$symrefcap" != "$headref" ] &&
 533                                    { [ -n "$symrefcap" ] || [ $matchcnt -gt 1 ]; }; then
 534                                         showheadwarn=1
 535                                 fi
 536                         fi
 537                         if [ -z "$headref" ]; then
 538                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 539                                 # if it exists otherwise take the first refs/heads/...
 540                                 # We do not support having a detached HEAD.
 541                                 # We always warn now because we will be setting HEAD differently
 542                                 # than the source repository had HEAD set
 543                                 showheadwarn=1
 544                                 while read ref; do
 545                                         [ -n "$ref" ] || continue
 546                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 547                                                 headref="$ref"
 548                                         fi
 549                                         [ "$headref" != "refs/heads/master" ] || break
 550                                 done <<-EOT
 551                                 $(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 552                                         LC_ALL=C awk '{print $2}')
 553                                 EOT
 554                         fi
 555                         # If we STILL do not have a HEAD ref (perhaps the source repository
 556                         # contains only tags) then use refs/heads/master.  It will be invalid
 557                         # but is no worse than we used to do by default and we'll warn about
 558                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 559                         [ -n "$headref" ] || headref="refs/heads/master"
 560                         git symbolic-ref HEAD "$headref"
 561                         pruneopt=--prune
 562                         [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
 563                         # remember the starting time so we can easily detect new packs for fast-import mirrors
 564                         # we sleep for 1 second after creating .gfipack to make sure all packs are newer
 565                         if is_gfi_mirror_url "$url" && ! [ -e .gfipack ]; then
 566                                 rm -f .gfipack
 567                                 >.gfipack
 568                                 sleep 1
 569                         fi
 570                         GIT_SSL_NO_VERIFY=1 git remote update $pruneopt
 571                         if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
 572                                 find -L objects/pack -type f -newer .gfipack -name "pack-$octet20*.pack" -print >>gfi-packs
 573                                 rm -f .gfipack
 574                         fi
 575                 else
 576                         warnempty=1
 577                         git symbolic-ref HEAD "refs/heads/master"
 578                 fi
 579                 rm -f .refs-temp .pkts-temp
 580                 ;;
 581 esac
 582
 583 # The objects subdirectories permissions must be updated now.
 584 # In the case of a dumb http clone, the permissions will not be correct
 585 # (missing group write) despite the core.sharedrepository=1 setting!
 586 # The objects themselves seem to have the correct permissions.
 587 # This problem appears to have been fixed in the most recent git versions.
 588 perms=g+w
 589 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 590 chmod $perms $(find -L objects -maxdepth 1 -type d) 2>/dev/null || :
 591
 592 # We may have just cloned a lot of refs and they will all be
 593 # individual files at this point.  Let's pack them now so we
 594 # can have better performance right from the start.
 595 git pack-refs --all
 596
 597 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 598 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 599 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 600 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 601         --count=1 refs/heads >info/lastactivity || :
 602 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 603
 604 # Don't leave a multi-megabyte useless FETCH_HEAD behind
 605 rm -f FETCH_HEAD
 606
 607 # Last ditch attempt to get a valid HEAD for a non-git source
 608 check_and_set_head || :
 609
 610 # The rest
 611 echo "Final touches..."
 612 git update-server-info
 613 trap "" EXIT
 614
 615 # run gc now unless the clone is empty
 616 if [ -z "$warnempty" ]; then
 617         git config --unset gitweb.lastgc 2>/dev/null || :
 618         rm -f .delaygc .allowgc
 619 fi
 620
 621 emptynote=
 622 [ -z "$warnempty" ] ||
 623 emptynote="
 624 WARNING:  You have mirrored an empty repository.
 625 "
 626 headnote=
 627 [ -z "$showheadwarn" ] || [ -z "$headref" ] ||
 628 headnote="
 629 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 630        Use the \"Project settings\" link to choose a different HEAD symref.
 631 "
 632 sizenote=
 633 ! is_gfi_mirror ||
 634 sizenote="
 635 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 636        size may be somewhat larger than necessary.  This will be corrected
 637        shortly.  If you intend to clone this repository you may want to
 638        wait up to 1 hour before doing so in order to receive the more
 639        compact final size.
 640 "
 641 [ -z "$mailaddrs" ] ||
 642 mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 643 Congratulations! The clone of project $proj just completed.
 644
 645         * Source URL: $url
 646         * GitWeb interface: $cfg_gitweburl/$proj.git
 647         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 648 $emptynote$headnote$sizenote
 649 Have a lot of fun.
 650 EOT
 651
 652 echo "Mirroring finished successfuly!"
 653 # In case this is a re-mirror, lastgc could have been set already so clear it now
 654 git config --unset gitweb.lastgc || :
 655 rm .clone_in_progress
 656 echo "$sizenote@OVER@"