taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6 . @basedir@/jobd/gc-util-functions.sh
   7
   8 set -e
   9
  10 umask 002
  11 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  12 clean_git_env
  13
  14 # darcs fast-export | git fast-import with error handling
  15 git_darcs_fetch() (
  16         set_utf8_locale
  17         _err1=
  18         _err2=
  19         exec 3>&1
  20         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  21         $(
  22                 exec 4>&3 3>&1 1>&4 4>&-
  23                 {
  24                         _e1=0
  25                         "$cfg_basedir"/bin/darcs-fast-export \
  26                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  27                         echo $_e1 >&3
  28                 } |
  29                 {
  30                         _e2=0
  31                         git_ulimit fast-import \
  32                                 --export-marks="$(pwd)/gfi-marks" \
  33                                 --export-pack-edges="$(pwd)/gfi-packs" \
  34                                 --force 3>&- || _e2=$?
  35                         echo $_e2 >&3
  36                 }
  37         )
  38         EOT
  39         exec 3>&-
  40         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  41         return $?
  42 )
  43
  44 # bzr fast-export | git fast-import with error handling
  45 git_bzr_fetch() (
  46         set_utf8_locale
  47         BZR_LOG=/dev/null
  48         export BZR_LOG
  49         _err1=
  50         _err2=
  51         exec 3>&1
  52         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  53         $(
  54                 exec 4>&3 3>&1 1>&4 4>&-
  55                 {
  56                         _e1=0
  57                         bzr fast-export --plain \
  58                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  59                         echo $_e1 >&3
  60                 } |
  61                 {
  62                         _e2=0
  63                         git_ulimit fast-import \
  64                                 --export-marks="$(pwd)/gfi-marks" \
  65                                 --export-pack-edges="$(pwd)/gfi-packs" \
  66                                 --force 3>&- || _e2=$?
  67                         echo $_e2 >&3
  68                 }
  69         )
  70         EOT
  71         exec 3>&-
  72         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  73         return $?
  74 )
  75
  76 clear_all_objects_and_packs() {
  77         if [ -d objects ]; then
  78                 # make sure the repository is not left broken
  79                 printf '%s\n' 'ref: refs/heads/master' >HEAD || :
  80                 rm -f packed-refs || :
  81                 find -H refs objects -type f -exec rm -f '{}' + >/dev/null 2>&1 || :
  82                 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
  83         fi
  84 }
  85
  86 exit_err=0
  87 send_clone_failed() {
  88         trap "" EXIT
  89         # We must now close the .clonelog file that is open on stdout and stderr
  90         exec >/dev/null 2>&1
  91         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
  92         failaddrs="$(config_get owner)" || :
  93         ccadm="${cfg_admincc:-0}"
  94         xfsz_err=""
  95         if [ "${exit_err:-0}" = "${var_xfsz_err:-999}" ]; then
  96                 ccadm=1
  97                 reposize="$(cd objects && du -sk . | LC_ALL=C awk '{print $1}')" || :
  98                 if [ -n "$reposize" ]; then
  99                         if [ $reposize -lt 5120 ]; then
 100                                 reposize="$reposize KiB"
 101                         else
 102                                 reposize="$(( $reposize / 1024 ))"
 103                                 if [ $reposize -lt 5120 ]; then
 104                                         reposize="$reposize MiB"
 105                                 else
 106                                         reposize="$(( $reposize / 1024 ))"
 107                                         reposize="$reposize GiB"
 108                                 fi
 109                         fi
 110                 fi
 111                 xfsz_err="
 112
 113 The source repository${reposize:+ ($reposize)} exceeds our maximum allowed repository size."
 114                 clear_all_objects_and_packs
 115         fi
 116         [ "$ccadm" = "0" ] || [ -z "$cfg_admin" ] ||
 117         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
 118         [ -z "$failaddrs" ] ||
 119         {
 120               cat <<EOT
 121 Condolences. The clone of project $proj just failed.$xfsz_err
 122
 123         * Source URL: $url
 124         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 125
 126 The project settings link may be used to adjust the settings
 127 and restart the clone in order to try the clone again.
 128 EOT
 129                 if [ -f .clonelog ] && [ -r .clonelog ]; then
 130                         echo ""
 131                         echo "Log follows:"
 132                         echo ""
 133                         loglines=$(LC_ALL=C wc -l <.clonelog)
 134                         if [ $loglines -le 203 ]; then
 135                                 cat .clonelog
 136                         else
 137                                 head -n 100 .clonelog
 138                                 echo ""
 139                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
 140                                 echo ""
 141                                 tail -n 100 .clonelog
 142                         fi
 143                 fi
 144         } | mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 145 }
 146
 147 # removes any git-svn leftovers
 148 cleanup_git_svn_leftovers() {
 149
 150         # Remove any stale git-svn temp files
 151         # The git-svn process creates temp files with random 10 character names
 152         # in the root of $GIT_DIR.  Unfortunately they do not have a recognizable
 153         # prefix, so we just have to kill any files with a 10-character name.
 154         # All characters are chosen from
 155         # [A-Za-z0-9_] so we can at least check that and fortunately the only
 156         # collision is 'FETCH_HEAD' but that doesn't matter.
 157         # There may also be temp files with a Git_ prefix as well.
 158         _randchar='[A-Za-z0-9_]'
 159         _randchar2="$_randchar$_randchar"
 160         _randchar4="$_randchar2$_randchar2"
 161         _randchar10="$_randchar4$_randchar4$_randchar2"
 162         find -L . -maxdepth 1 -type f -name "$_randchar10" -exec rm -f '{}' + || :
 163         find -L . -maxdepth 1 -type f -name "Git_*" -exec rm -f '{}' + || :
 164 }
 165
 166 # removes all leftovers from a previous failed clone attempt
 167 cleanup_failed_clone() {
 168
 169         # Remove any left-over svn-remote.svn or remote.origin config
 170         git config --remove-section svn-remote.svn 2>/dev/null || :
 171         git config --remove-section remote.origin 2>/dev/null || :
 172
 173         # If there is a remote-template.origin section, pre-seed the
 174         # remote.origin section with its contents
 175         git config --get-regexp '^remote-template\.origin\..' |
 176         while read name value; do
 177                 if [ -n "$name" ] && [ -n "$value" ]; then
 178                         git config "remote${name#remote-template}" "$value"
 179                 fi
 180         done
 181
 182         # Any pre-existing FETCH_HEAD from a previous clone failed or not is
 183         # now garbage to be removed
 184         rm -f FETCH_HEAD
 185
 186         # Remove any stale ref locks
 187         clear_stale_ref_locks
 188
 189         # Remove any left-over svn dir from a previous failed attempt
 190         rm -rf svn
 191
 192         # Remove any left-over .darcs dirs from a previous failed attempt
 193         rm -rf *.darcs
 194
 195         # Remove any left-over repo.hg dir from a previous failed attempt
 196         rm -rf repo.hg
 197
 198         # Remove any left-over import/export/temp files from a previous failed attempt
 199         rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks* hg2git-state \
 200                 gfi-marks gfi-packs .pkts-temp .refs-temp
 201
 202         # Remove any git-svn junk
 203         cleanup_git_svn_leftovers
 204
 205         # We want a gc right after the clone, so re-enable that just in case.
 206         # There's a potential race where we could add it and gc.sh could remove
 207         # it, but we'll reunset lastgc just before we remove .delaygc at the end.
 208         [ -e .delaygc ] || >.delaygc
 209         git config --unset gitweb.lastgc 2>/dev/null || :
 210
 211         # Remove all pre-existing refs
 212         rm -f packed-refs
 213         git for-each-ref --format='delete %(refname)' | git_updateref_stdin 2>/dev/null || :
 214
 215         # The initial state before a clone starts has HEAD as a symbolic-ref to master
 216         git symbolic-ref HEAD refs/heads/master
 217
 218         # HEAD is no longer "ok"
 219         git config --unset girocco.headok 2>/dev/null || :
 220
 221         # We, perhaps, ought to remove any packs/loose objects now, but the next gc
 222         # will get rid of any extras.  Also, if we're recloning the same thing, any
 223         # preexisting packs/loose objects containing what we're recloning will only
 224         # speed up the reclone by avoiding some disk writes.  So we don't kill them.
 225
 226         # It's just remotely possible that a bunch of failures in a row could
 227         # create a big mess that just keeps growing and growing...
 228         # Trigger a .needsgc if that happens.
 229         check_and_set_needsgc
 230 }
 231
 232 proj="${1%.git}"
 233 cd "$cfg_reporoot/$proj.git"
 234 bang_reset
 235
 236 ! [ -e .delaygc ] || >.allowgc || :
 237
 238 trap "exit_err=$?; echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 239 echo "Project: $proj"
 240 echo "   Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
 241 echo ""
 242 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 243 url="$(config_get baseurl)" || :
 244 case "$url" in *"       "*|*" "*|"")
 245         echo "Bad mirror URL (\"$url\")"
 246         exit 1
 247 esac
 248
 249 cleanup_failed_clone
 250
 251 # Record original mirror type for use by update.sh
 252 mirror_type="$(get_url_mirror_type "$url")"
 253 git config girocco.mirrortype "$mirror_type"
 254
 255 echo "Mirroring from URL \"$url\""
 256 echo ""
 257
 258 if [ "$cfg_project_owners" = "source" ]; then
 259         config set owner "$(ls -ldH "${url#file://}" 2>/dev/null | LC_ALL=C awk '{print $3}')"
 260 fi
 261
 262 mailaddrs="$(config_get owner)" || :
 263 [ -z "$cfg_admin" ] ||
 264 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 265
 266 # Make sure we don't get any unwanted loose objects
 267 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
 268 # tweak its configuration to prevent that
 269 git_add_config 'fetch.unpackLimit=1'
 270 # Note the git config documentation is wrong
 271 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
 272 git_add_config 'transfer.unpackLimit=1'
 273 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
 274 git_add_config 'fastimport.unpackLimit=0'
 275
 276 # Initial mirror
 277 echo "Initiating mirroring..."
 278 headref=
 279 showheadwarn=
 280 warnempty=
 281
 282 # remember the starting time so we can easily combine fetched loose objects
 283 # we sleep for 1 second after creating .needspack to make sure all objects are newer
 284 if ! [ -e .needspack ]; then
 285         rm -f .needspack
 286         >.needspack
 287         sleep 1
 288 fi
 289
 290 case "$url" in
 291         svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 292                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 293                 # Allow the username to be specified in the "svn-credential.svn.username"
 294                 # property and the password in the "svn-credential.svn.password" property
 295                 # Use an 'anonsvn' username by default as is commonly used for anonymous svn
 296                 # Default the password to the same as the username
 297                 # The password property will be ignored unless a username has been specified
 298                 if svnuser="$(git config --get svn-credential.svn.username)" && [ -n "$svnuser" ]; then
 299                         if ! svnpass="$(git config --get svn-credential.svn.password)"; then
 300                                 svnpass="$svnuser"
 301                         fi
 302                         url1="${url#*://}"
 303                         url1="${url1%%/*}"
 304                         case "$url1" in ?*"@"?*)
 305                                 urlsch="${url%%://*}"
 306                                 url="$urlsch://${url#*@}"
 307                         esac
 308                 else
 309                         # As a fallback, check in the URL, just in case
 310                         url1="${url#*://}"
 311                         url1="${url1%%/*}"
 312                         svnuser=
 313                         case "$url1" in ?*"@"?*)
 314                                 urlsch="${url%%://*}"
 315                                 url="$urlsch://${url#*@}"
 316                                 url1="${url1%%@*}"
 317                                 svnuser="${url1%%:*}"
 318                                 if [ -n "$svnuser" ]; then
 319                                         svnpass="$svnuser"
 320                                         case "$url1" in *":"*)
 321                                                 svnpass="${url1#*:}"
 322                                         esac
 323                                 fi
 324                         esac
 325                         if [ -z "$svnuser" ]; then
 326                                 svnuser="anonsvn"
 327                                 svnpass="anonsvn"
 328                         fi
 329                 fi
 330                 GIT_ASKPASS_PASSWORD="$svnpass"
 331                 export GIT_ASKPASS_PASSWORD
 332                 # We just remove svn+ here, so svn+http://... becomes http://...
 333                 # We also remove a trailing '/' to match what git-svn will do
 334                 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
 335                 svnurl="${svnurl%/}"
 336                 # We require svn info to succeed on the URL otherwise it's
 337                 # simply not a valid URL and without using -s on the init it
 338                 # will not otherwise be tested until the fetch
 339                 svn --non-interactive --username "$svnuser" --password "$svnpass" info "$svnurl" >/dev/null
 340                 # We initially use -s for the init which will possibly shorten
 341                 # the URL.  However, the shortening can fail if a password is
 342                 # not required for the longer version but is for the shorter,
 343                 # so try again without -s if the -s version fails.
 344                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 345                 # create a .git subdirectory!
 346                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" -s "$svnurl" <"$mtlinesfile" ||
 347                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" "$svnurl" <"$mtlinesfile"
 348                 # We need to remember this url so we can detect changes because
 349                 # ever so "helpful" git-svn may shorten it!
 350                 config_set svnurl "$svnurl"
 351                 # At this point, since we asked for a standard layout (-s) git-svn
 352                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 353                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 354                 # config items.  We could avoid this by not using the '-s' option
 355                 # but then we might not get all the history.  If, for example, we
 356                 # are cloning an http://svn.example.com/repos/public repository that
 357                 # early in its history moved trunk => public/trunk we would miss that
 358                 # earlier history without allowing the funky shorten+prefix behavior.
 359                 # So we read back the svn-remote.svn.fetch configuration and compute
 360                 # the prefix.  This way we are sure to get the correct prefix.
 361                 gitsvnurl="$(git config --get svn-remote.svn.url)" || :
 362                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1)" || :
 363                 gitsvnprefix="${gitsvnfetch%%:*}"
 364                 gitsvnsuffix="${gitsvnprefix##*/}"
 365                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 366                 # Ask git-svn to store everything in the normal non-remote
 367                 # locations being careful to use the correct prefix
 368                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 369                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 370                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 371                 # look for additional non-standard directories to fetch
 372                 # check for standard layout at the same time
 373                 foundstd=
 374                 foundfile=
 375                 svn --non-interactive --username "$svnuser" --password "$svnpass" ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null |
 376                 { while read file; do case $file in
 377                         # skip the already-handled standard ones and any with a space or tab
 378                         *'      '*|*' '*) :;;
 379                         trunk/|branches/|tags/) foundstd=1;;
 380                         # only fetch extra directories from the $svnurl root (not any files)
 381                         *?/) git config --add svn-remote.svn.fetch \
 382                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 383                         *?) foundfile=1;;
 384                 esac; done
 385                 # if files found and no standard directories present use a simpler layout
 386                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 387                         git config --unset svn-remote.svn.branches
 388                         git config --unset svn-remote.svn.tags
 389                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 390                 fi; }
 391                 test $? -eq 0
 392                 # git svn fetch on a very large repo can take some time and the
 393                 # remote server may interrupt the connection from time to time.
 394                 # keep retrying (after a brief pause) as long as we are making progress.
 395                 # however, we do limit the total number of retries to 1000
 396                 # we will, however, retry up to 5 times even if we're not making progress
 397                 v_get_svn_progress_fingerprint() {
 398                         eval "$1="'"$({ GIT_DIR=. git svn info <"$mtlinesfile" 2>&1; git show-ref --head 2>&1; } |
 399                         git hash-object -t blob --stdin )"' || :
 400                 }
 401                 svn_ret_err() { return "${1:-1}"; }
 402                 svn_retries=1000 # maximum possible fetch attempts no matter what
 403                 svn_retry_backoff_start_half=60 # min retry wait is double this amount in seconds
 404                 svn_backoff_count=7 # max retry wait is $svn_retry_backoff_start_half * 2^$svn_backoff_count
 405                 # Cumulative backoff wait before giving up on consecutive no-progress retries
 406                 #   is approximately 2 * $svn_retry_backoff_start_half * 2^$svn_backoff_count
 407                 # For a $svn_backoff_count of 7 that works out to be exactly 4h14m
 408                 svn_progress=
 409                 v_get_svn_progress_fingerprint svn_progress
 410                 svn_progress_retries="$svn_retries"
 411                 svn_retry_backoff="$svn_retry_backoff_start_half"
 412                 svn_err=0
 413                 while [ "$svn_retries" -gt 0 ]; do
 414                         svn_retries="$(( $svn_retries - 1 ))"
 415                         svn_err=0
 416                         GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1
 417                         export GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 418                         unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 419                         saveconfig="$GIT_CONFIG_PARAMETERS"
 420                         git_add_config 'gc.auto=1'
 421                         git_add_config 'gc.autoPackLimit=1'
 422                         # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 423                         GIT_DIR=. git_ulimit svn fetch --log-window-size=$var_log_window_size --username="$svnuser" --quiet <"$mtlinesfile" || svn_err="$?"
 424                         GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 425                         export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 426                         unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 427                         unset GIT_CONFIG_PARAMETERS
 428                         [ -z "$saveconfig" ] || {
 429                                 GIT_CONFIG_PARAMETERS="$saveconfig"
 430                                 export GIT_CONFIG_PARAMETERS
 431                         }
 432                         [ "${svn_err:-1}" -ne 0 ] || break # success!
 433                         # Check to see if we made any progress
 434                         v_get_svn_progress_fingerprint svn_progress_now
 435                         if [ "$svn_progress_now" != "$svn_progress" ]; then
 436                                 # we made progress, continue the loop with min wait
 437                                 svn_progress="$svn_progress_now"
 438                                 svn_progress_retries="$svn_retries"
 439                                 svn_retry_backoff="$svn_retry_backoff_start_half"
 440                         else
 441                                 # no progress, but we only give up after
 442                                 # $svn_backoff_count no-progress attempts in a row
 443                                 [ "$(( $svn_progress_retries - $svn_retries ))" -lt "$svn_backoff_count" ] ||
 444                                         break # failure
 445                                 # continue but only after twice the previous wait
 446                                 # (which will still be the min wait if this is the
 447                                 # first no-progress retry after making some progress)
 448                         fi
 449                         svn_retry_backoff="$(( 2 * $svn_retry_backoff ))"
 450                         # Pause for $svn_retry_backoff seconds before retrying to be friendly to the server
 451                         # Use that time to pack up loose objects if there are "lotsa" them
 452                         if ! lotsa_loose_objects_or_sopacks; then
 453                                 echo "Pausing for $svn_retry_backoff seconds before retrying ($(date))"
 454                                 sleep "$svn_retry_backoff"
 455                         else
 456                                 pausestop="$(( $(date '+%s') + $svn_retry_backoff ))"
 457                                 echo "Pausing and packing loose objects for $svn_retry_backoff seconds before retrying ($(date))"
 458                                 pack_incremental_loose_objects_if_lockable ||
 459                                 echo "Packing skipped (only pausing): $lockerr"
 460                                 timenow="$(date '+%s')"
 461                                 if [ "$timenow" -lt "$pausestop" ]; then
 462                                         sleepamt="$(( $pausestop - $timenow ))"
 463                                         [ "$sleepamt" -le "$svn_retry_backoff" ] ||
 464                                                 sleepamt="$svn_retry_backoff" # paranoia check
 465                                         sleep "$sleepamt"
 466                                 fi
 467                         fi
 468                         cleanup_git_svn_leftovers
 469                         echo "Retrying fetch ($(date))"
 470                 done
 471                 [ "${svn_err:-1}" -eq 0 ] || svn_ret_err "$svn_err"
 472                 test ${svn_err:-1} -eq 0
 473                 # git svn does not preserve group permissions in the svn subdirectory
 474                 chmod -R ug+rw,o+r svn
 475                 # git svn also leaves behind ref turds that end with @nnn
 476                 # We get rid of them now
 477                 git for-each-ref --format='%(refname)' |
 478                 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
 479                 git_updateref_stdin
 480                 unset GIT_ASKPASS_PASSWORD
 481                 ;;
 482         darcs://* | darcs+http://* | darcs+https://*)
 483                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 484                 case "$url" in
 485                         darcs://*) darcsurl="http://${url#darcs://}";;
 486                         *) darcsurl="${url#darcs+}";;
 487                 esac
 488                 git_darcs_fetch "$darcsurl"
 489                 ;;
 490         bzr://*)
 491                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 492                 # we just remove bzr:// here, a typical bzr url is just
 493                 # "lp:foo"
 494                 bzrurl="${url#bzr://}"
 495                 git_bzr_fetch "$bzrurl"
 496                 ;;
 497         hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
 498                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 499                 # We just remove hg+ here, so hg+http://... becomes http://...
 500                 hgurl="${url#hg+}"
 501                 # Perform the initial hg clone
 502                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 503                 # Do the fast-export | fast-import
 504                 git_hg_fetch
 505                 ;;
 506         *)
 507                 # We manually add remote.origin.url and remote.origin.fetch
 508                 # to simulate a `git remote add --mirror=fetch` since that's
 509                 # not available until Git 1.7.5 and this way we guarantee we
 510                 # always get exactly the intended configuration and nothing else.
 511                 git config remote.origin.url "$url"
 512                 if ! is_gfi_mirror_url "$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
 513                         git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
 514                         git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
 515                         git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
 516                         git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
 517                         git config --bool girocco.lastupdateclean true
 518                 else
 519                         git config --replace-all remote.origin.fetch "+refs/*:refs/*"
 520                         git config --bool girocco.lastupdateclean false
 521                 fi
 522                 # Set the correct HEAD symref by using ls-remote first
 523                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp ||
 524                 {
 525                         # Since everything was redirected, on failure there'd be no output,
 526                         # so let's make some failure output
 527                         cat .pkts-temp
 528                         echo ""
 529                         echo "git ls-remote \"$url\" failed"
 530                         exit 1
 531                 }
 532                 # Compensate for git() {} side effects
 533                 unset GIT_TRACE_PACKET
 534                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 535                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 536                 # out of the packet trace data.
 537                 if [ -s .refs-temp ]; then
 538                         # Nothing to do unless the remote repository has at least 1 ref
 539                         # See if we got a HEAD ref
 540                         head="$(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+HEAD\$" <.refs-temp | LC_ALL=C awk '{print $1}')"
 541                         # If the remote has HEAD set to a symbolic ref that does not exist
 542                         # then we will not receive a HEAD ref in the ls-remote output
 543                         headref=
 544                         showheadwarn=
 545                         symrefcap=
 546                         if [ -n "$head" ]; then
 547                                 symrefcap="$(LC_ALL=C sed -ne <.pkts-temp \
 548                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 549                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 550                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 551                                 # matches HEAD otherwise take the first refs/heads/... match
 552                                 matchcnt=0
 553                                 while read ref; do
 554                                         [ -n "$ref" ] || continue
 555                                         matchcnt=$(( $matchcnt + 1 ))
 556                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 557                                                 headref="$ref"
 558                                         fi
 559                                         if [ "$headref" = "${symrefcap:-refs/heads/master}" ] && [ $matchcnt -gt 1 ]; then
 560                                                 break
 561                                         fi
 562                                 done <<-EOT
 563                                 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 564                                         LC_ALL=C awk '{print $2}')
 565                                 EOT
 566                                 # Warn if there was more than one match and $symrefcap is empty
 567                                 # or $symrefcap is not the same as $headref since our choice might
 568                                 # differ from the source repository's HEAD
 569                                 if [ $matchcnt -ge 1 ] && [ "$symrefcap" != "$headref" ] &&
 570                                    { [ -n "$symrefcap" ] || [ $matchcnt -gt 1 ]; }; then
 571                                         showheadwarn=1
 572                                 fi
 573                         fi
 574                         if [ -z "$headref" ]; then
 575                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 576                                 # if it exists otherwise take the first refs/heads/...
 577                                 # We do not support having a detached HEAD.
 578                                 # We always warn now because we will be setting HEAD differently
 579                                 # than the source repository had HEAD set
 580                                 showheadwarn=1
 581                                 while read ref; do
 582                                         [ -n "$ref" ] || continue
 583                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 584                                                 headref="$ref"
 585                                         fi
 586                                         [ "$headref" != "refs/heads/master" ] || break
 587                                 done <<-EOT
 588                                 $(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 589                                         LC_ALL=C awk '{print $2}')
 590                                 EOT
 591                         fi
 592                         # If we STILL do not have a HEAD ref (perhaps the source repository
 593                         # contains only tags) then use refs/heads/master.  It will be invalid
 594                         # but is no worse than we used to do by default and we'll warn about
 595                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 596                         [ -n "$headref" ] || headref="refs/heads/master"
 597                         git symbolic-ref HEAD "$headref"
 598                         pruneopt=--prune
 599                         [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
 600                         # remember the starting time so we can easily detect new packs for fast-import mirrors
 601                         # we sleep for 1 second after creating .gfipack to make sure all packs are newer
 602                         if is_gfi_mirror_url "$url" && ! [ -e .gfipack ]; then
 603                                 rm -f .gfipack
 604                                 >.gfipack
 605                                 sleep 1
 606                         fi
 607                         GIT_SSL_NO_VERIFY=1 git_ulimit remote update $pruneopt
 608                         if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
 609                                 find -L objects/pack -type f -newer .gfipack -name "pack-$octet20*.pack" -print >>gfi-packs
 610                                 rm -f .gfipack
 611                         fi
 612                 else
 613                         warnempty=1
 614                         git symbolic-ref HEAD "refs/heads/master"
 615                 fi
 616                 rm -f .refs-temp .pkts-temp
 617                 ;;
 618 esac
 619
 620 # The objects subdirectories permissions must be updated now.
 621 # In the case of a dumb http clone, the permissions will not be correct
 622 # (missing group write) despite the core.sharedrepository=1 setting!
 623 # The objects themselves seem to have the correct permissions.
 624 # This problem appears to have been fixed in the most recent git versions.
 625 perms=g+w
 626 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 627 chmod $perms $(find -L objects -maxdepth 1 -type d) 2>/dev/null || :
 628
 629 # We may have just cloned a lot of refs and they will all be
 630 # individual files at this point.  Let's pack them now so we
 631 # can have better performance right from the start.
 632 git pack-refs --all
 633
 634 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 635 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 636 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 637 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 638         --count=1 refs/heads >info/lastactivity || :
 639 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 640
 641 # Don't leave a multi-megabyte useless FETCH_HEAD behind
 642 rm -f FETCH_HEAD
 643
 644 # Last ditch attempt to get a valid HEAD for a non-git source
 645 check_and_set_head || :
 646
 647 # The rest
 648 echo "Final touches..."
 649 git update-server-info
 650 trap "" EXIT
 651
 652 # run gc now unless the clone is empty
 653 if [ -z "$warnempty" ]; then
 654         git config --unset gitweb.lastgc 2>/dev/null || :
 655         rm -f .delaygc .allowgc
 656 fi
 657
 658 emptynote=
 659 [ -z "$warnempty" ] ||
 660 emptynote="
 661 WARNING:  You have mirrored an empty repository.
 662 "
 663 headnote=
 664 [ -z "$showheadwarn" ] || [ -z "$headref" ] ||
 665 headnote="
 666 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 667        Use the \"Project settings\" link to choose a different HEAD symref.
 668 "
 669 sizenote=
 670 ! is_gfi_mirror ||
 671 sizenote="
 672 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 673        size may be somewhat larger than necessary.  This will be corrected
 674        shortly.  If you intend to clone this repository you may want to
 675        wait up to 1 hour before doing so in order to receive the more
 676        compact final size.
 677 "
 678 [ -z "$mailaddrs" ] ||
 679 mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 680 Congratulations! The clone of project $proj just completed.
 681
 682         * Source URL: $url
 683         * GitWeb interface: $cfg_gitweburl/$proj.git
 684         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 685 $emptynote$headnote$sizenote
 686 Have a lot of fun.
 687 EOT
 688
 689 echo "Mirroring finished successfuly!"
 690 # In case this is a re-mirror, lastgc could have been set already so clear it now
 691 git config --unset gitweb.lastgc || :
 692 rm .clone_in_progress
 693 echo "$sizenote@OVER@"