taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6 . @basedir@/jobd/gc-util-functions.sh
   7
   8 set -e
   9
  10 umask 002
  11 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  12 clean_git_env
  13
  14 # darcs fast-export | git fast-import with error handling
  15 git_darcs_fetch() (
  16         set_utf8_locale
  17         _err1=
  18         _err2=
  19         exec 3>&1
  20         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  21         $(
  22                 exec 4>&3 3>&1 1>&4 4>&-
  23                 {
  24                         _e1=0
  25                         "$cfg_basedir"/bin/darcs-fast-export \
  26                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  27                         echo $_e1 >&3
  28                 } |
  29                 {
  30                         _e2=0
  31                         git_ulimit fast-import \
  32                                 --export-marks="$(pwd)/gfi-marks" \
  33                                 --export-pack-edges="$(pwd)/gfi-packs" \
  34                                 --force 3>&- || _e2=$?
  35                         echo $_e2 >&3
  36                 }
  37         )
  38         EOT
  39         exec 3>&-
  40         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  41         return $?
  42 )
  43
  44 # bzr fast-export | git fast-import with error handling
  45 git_bzr_fetch() (
  46         set_utf8_locale
  47         BZR_LOG=/dev/null
  48         export BZR_LOG
  49         _err1=
  50         _err2=
  51         exec 3>&1
  52         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  53         $(
  54                 exec 4>&3 3>&1 1>&4 4>&-
  55                 {
  56                         _e1=0
  57                         bzr fast-export --plain \
  58                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  59                         echo $_e1 >&3
  60                 } |
  61                 {
  62                         _e2=0
  63                         git_ulimit fast-import \
  64                                 --export-marks="$(pwd)/gfi-marks" \
  65                                 --export-pack-edges="$(pwd)/gfi-packs" \
  66                                 --force 3>&- || _e2=$?
  67                         echo $_e2 >&3
  68                 }
  69         )
  70         EOT
  71         exec 3>&-
  72         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  73         return $?
  74 )
  75
  76 clear_all_objects_and_packs() {
  77         if [ -d objects ]; then
  78                 # make sure the repository is not left broken
  79                 printf '%s\n' 'ref: refs/heads/master' >HEAD || :
  80                 rm -f packed-refs || :
  81                 find -H refs objects -type f -exec rm -f '{}' + >/dev/null 2>&1 || :
  82                 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
  83         fi
  84 }
  85
  86 exit_err=0
  87 exit_objs=0
  88 send_clone_failed() {
  89         trap "" EXIT
  90         # We must now close the .clonelog file that is open on stdout and stderr
  91         exec >/dev/null 2>&1
  92         failaddrs="$(config_get owner)" || :
  93         ccadm="${cfg_admincc:-0}"
  94         xfsz_err=""
  95         if [ "${exit_err:-0}" = "${var_xfsz_err:-999}" ]; then
  96                 ccadm=1
  97                 reposize="$(cd objects && du -sk . | LC_ALL=C awk '{print $1}')" || :
  98                 if [ -n "$reposize" ]; then
  99                         if [ $reposize -lt 5120 ]; then
 100                                 reposize="$reposize KiB"
 101                         else
 102                                 reposize="$(( $reposize / 1024 ))"
 103                                 if [ $reposize -lt 5120 ]; then
 104                                         reposize="$reposize MiB"
 105                                 else
 106                                         reposize="$(( $reposize / 1024 ))"
 107                                         reposize="$reposize GiB"
 108                                 fi
 109                         fi
 110                 fi
 111                 xfsz_err="
 112
 113 The source repository${reposize:+ ($reposize)} exceeds our maximum allowed repository size."
 114                 clear_all_objects_and_packs
 115         fi
 116         xobjs_err=""
 117         if [ "${exit_objs:-0}" != "0" ]; then
 118                 ccadm=1
 119                 xobjs_err="
 120
 121 The source repository${exit_objs:+ ($exit_objs objects)} exceeds our maximum allowed object limit."
 122                 clear_all_objects_and_packs
 123         fi
 124         if [ -n "$xfsz_err" ] || [ -n "$xobjs_err" ]; then
 125                 # Mark as an exceeds limit clone failure and remember the exceeds
 126                 # message(s) in both .clone_failed_exceeds_limit and .clonelog
 127                 >.clone_failed_exceeds_limit
 128                 if [ -n "$xfsz_err" ]; then
 129                         printf '%s\n' "${xfsz_err#??}" >>.clone_failed_exceeds_limit
 130                         printf '%s\n' "${xfsz_err#?}" >>.clonelog
 131                 fi
 132                 if [ -n "$xobjs_err" ]; then
 133                         printf '%s\n' "${xobjs_err#??}" >>.clone_failed_exceeds_limit
 134                         printf '%s\n' "${xobjs_err#?}" >>.clonelog
 135                 fi
 136                 # Remove the .clone_failed file to prevent "restarting" the clone since
 137                 # restarting it will not cure the fact that it exceeds allowed limits
 138                 # And the .clone_in_progress file has to go at the same time
 139                 rm -f .clone_in_progress .clone_failed
 140         fi
 141         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 142         [ "$ccadm" = "0" ] || [ -z "$cfg_admin" ] ||
 143         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
 144         [ -z "$failaddrs" ] ||
 145         {
 146               cat <<EOT
 147 Condolences. The clone of project $proj just failed.$xfsz_err$xobjs_err
 148
 149         * Source URL: $url
 150         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 151
 152 The project settings link may be used to adjust the settings
 153 and restart the clone in order to try the clone again.
 154 EOT
 155                 if [ -f .clonelog ] && [ -r .clonelog ]; then
 156                         echo ""
 157                         echo "Log follows:"
 158                         echo ""
 159                         loglines=$(LC_ALL=C wc -l <.clonelog)
 160                         if [ $loglines -le 203 ]; then
 161                                 cat .clonelog
 162                         else
 163                                 head -n 100 .clonelog
 164                                 echo ""
 165                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
 166                                 echo ""
 167                                 tail -n 100 .clonelog
 168                         fi
 169                 fi
 170         } | mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 171 }
 172
 173 # removes any git-svn leftovers
 174 cleanup_git_svn_leftovers() {
 175
 176         # Remove any stale git-svn temp files
 177         # The git-svn process creates temp files with random 10 character names
 178         # in the root of $GIT_DIR.  Unfortunately they do not have a recognizable
 179         # prefix, so we just have to kill any files with a 10-character name.
 180         # All characters are chosen from
 181         # [A-Za-z0-9_] so we can at least check that and fortunately the only
 182         # collision is 'FETCH_HEAD' but that doesn't matter.
 183         # There may also be temp files with a Git_ prefix as well.
 184         _randchar='[A-Za-z0-9_]'
 185         _randchar2="$_randchar$_randchar"
 186         _randchar4="$_randchar2$_randchar2"
 187         _randchar10="$_randchar4$_randchar4$_randchar2"
 188         find -L . -maxdepth 1 -type f -name "$_randchar10" -exec rm -f '{}' + || :
 189         find -L . -maxdepth 1 -type f -name "Git_*" -exec rm -f '{}' + || :
 190 }
 191
 192 # removes all leftovers from a previous failed clone attempt
 193 cleanup_failed_clone() {
 194
 195         # Remove any left-over svn-remote.svn or remote.origin config
 196         git config --remove-section svn-remote.svn 2>/dev/null || :
 197         git config --remove-section remote.origin 2>/dev/null || :
 198
 199         # If there is a remote-template.origin section, pre-seed the
 200         # remote.origin section with its contents
 201         git config --get-regexp '^remote-template\.origin\..' |
 202         while read name value; do
 203                 if [ -n "$name" ] && [ -n "$value" ]; then
 204                         git config "remote${name#remote-template}" "$value"
 205                 fi
 206         done
 207
 208         # Any pre-existing FETCH_HEAD from a previous clone failed or not is
 209         # now garbage to be removed
 210         rm -f FETCH_HEAD
 211
 212         # Remove any stale ref locks
 213         clear_stale_ref_locks
 214
 215         # Remove any left-over svn dir from a previous failed attempt
 216         rm -rf svn
 217
 218         # Remove any left-over .darcs dirs from a previous failed attempt
 219         rm -rf *.darcs
 220
 221         # Remove any left-over repo.hg dir from a previous failed attempt
 222         rm -rf repo.hg
 223
 224         # Remove any left-over import/export/temp files from a previous failed attempt
 225         rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks* hg2git-state \
 226                 gfi-marks gfi-packs .pkts-temp .refs-temp
 227
 228         # Remove any git-svn junk
 229         cleanup_git_svn_leftovers
 230
 231         # We want a gc right after the clone, so re-enable that just in case.
 232         # There's a potential race where we could add it and gc.sh could remove
 233         # it, but we'll reunset lastgc just before we remove .delaygc at the end.
 234         [ -e .delaygc ] || >.delaygc
 235         git config --unset gitweb.lastgc 2>/dev/null || :
 236
 237         # Remove all pre-existing refs
 238         rm -f packed-refs
 239         git for-each-ref --format='delete %(refname)' | git_updateref_stdin 2>/dev/null || :
 240
 241         # The initial state before a clone starts has HEAD as a symbolic-ref to master
 242         git symbolic-ref HEAD refs/heads/master
 243
 244         # HEAD is no longer "ok"
 245         git config --unset girocco.headok 2>/dev/null || :
 246
 247         # We, perhaps, ought to remove any packs/loose objects now, but the next gc
 248         # will get rid of any extras.  Also, if we're recloning the same thing, any
 249         # preexisting packs/loose objects containing what we're recloning will only
 250         # speed up the reclone by avoiding some disk writes.  So we don't kill them.
 251
 252         # It's just remotely possible that a bunch of failures in a row could
 253         # create a big mess that just keeps growing and growing...
 254         # Trigger a .needsgc if that happens.
 255         check_and_set_needsgc
 256 }
 257
 258 proj="${1%.git}"
 259 cd "$cfg_reporoot/$proj.git"
 260 bang_reset
 261
 262 ! [ -e .delaygc ] || >.allowgc || :
 263
 264 trap "exit_err=$?; echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 265 echo "Project: $proj"
 266 echo "   Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
 267 echo ""
 268 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 269 url="$(config_get baseurl)" || :
 270 case "$url" in *"       "*|*" "*|"")
 271         echo "Bad mirror URL (\"$url\")"
 272         exit 1
 273 esac
 274
 275 cleanup_failed_clone
 276
 277 # Record original mirror type for use by update.sh
 278 mirror_type="$(get_url_mirror_type "$url")"
 279 git config girocco.mirrortype "$mirror_type"
 280
 281 echo "Mirroring from URL \"$url\""
 282 echo ""
 283
 284 if [ "$cfg_project_owners" = "source" ]; then
 285         config set owner "$(ls -ldH "${url#file://}" 2>/dev/null | LC_ALL=C awk '{print $3}')"
 286 fi
 287
 288 mailaddrs="$(config_get owner)" || :
 289 [ -z "$cfg_admin" ] ||
 290 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 291
 292 # Make sure we don't get any unwanted loose objects
 293 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
 294 # tweak its configuration to prevent that
 295 git_add_config 'fetch.unpackLimit=1'
 296 # Note the git config documentation is wrong
 297 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
 298 git_add_config 'transfer.unpackLimit=1'
 299 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
 300 git_add_config 'fastimport.unpackLimit=0'
 301
 302 # Initial mirror
 303 echo "Initiating mirroring..."
 304 headref=
 305 showheadwarn=
 306 warnempty=
 307
 308 # remember the starting time so we can easily combine fetched loose objects
 309 # we sleep for 1 second after creating .needspack to make sure all objects are newer
 310 if ! [ -e .needspack ]; then
 311         rm -f .needspack
 312         >.needspack
 313         sleep 1
 314 fi
 315
 316 case "$url" in
 317         svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 318                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 319                 # Allow the username to be specified in the "svn-credential.svn.username"
 320                 # property and the password in the "svn-credential.svn.password" property
 321                 # Use an 'anonsvn' username by default as is commonly used for anonymous svn
 322                 # Default the password to the same as the username
 323                 # The password property will be ignored unless a username has been specified
 324                 if svnuser="$(git config --get svn-credential.svn.username)" && [ -n "$svnuser" ]; then
 325                         if ! svnpass="$(git config --get svn-credential.svn.password)"; then
 326                                 svnpass="$svnuser"
 327                         fi
 328                         url1="${url#*://}"
 329                         url1="${url1%%/*}"
 330                         case "$url1" in ?*"@"?*)
 331                                 urlsch="${url%%://*}"
 332                                 url="$urlsch://${url#*@}"
 333                         esac
 334                 else
 335                         # As a fallback, check in the URL, just in case
 336                         url1="${url#*://}"
 337                         url1="${url1%%/*}"
 338                         svnuser=
 339                         case "$url1" in ?*"@"?*)
 340                                 urlsch="${url%%://*}"
 341                                 url="$urlsch://${url#*@}"
 342                                 url1="${url1%%@*}"
 343                                 svnuser="${url1%%:*}"
 344                                 if [ -n "$svnuser" ]; then
 345                                         svnpass="$svnuser"
 346                                         case "$url1" in *":"*)
 347                                                 svnpass="${url1#*:}"
 348                                         esac
 349                                 fi
 350                         esac
 351                         if [ -z "$svnuser" ]; then
 352                                 svnuser="anonsvn"
 353                                 svnpass="anonsvn"
 354                         fi
 355                 fi
 356                 GIT_ASKPASS_PASSWORD="$svnpass"
 357                 export GIT_ASKPASS_PASSWORD
 358                 # We just remove svn+ here, so svn+http://... becomes http://...
 359                 # We also remove a trailing '/' to match what git-svn will do
 360                 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
 361                 svnurl="${svnurl%/}"
 362                 # We require svn info to succeed on the URL otherwise it's
 363                 # simply not a valid URL and without using -s on the init it
 364                 # will not otherwise be tested until the fetch
 365                 svn --non-interactive --username "$svnuser" --password "$svnpass" info "$svnurl" >/dev/null
 366                 # We initially use -s for the init which will possibly shorten
 367                 # the URL.  However, the shortening can fail if a password is
 368                 # not required for the longer version but is for the shorter,
 369                 # so try again without -s if the -s version fails.
 370                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 371                 # create a .git subdirectory!
 372                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" -s "$svnurl" <"$mtlinesfile" ||
 373                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" "$svnurl" <"$mtlinesfile"
 374                 # We need to remember this url so we can detect changes because
 375                 # ever so "helpful" git-svn may shorten it!
 376                 config_set svnurl "$svnurl"
 377                 # At this point, since we asked for a standard layout (-s) git-svn
 378                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 379                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 380                 # config items.  We could avoid this by not using the '-s' option
 381                 # but then we might not get all the history.  If, for example, we
 382                 # are cloning an http://svn.example.com/repos/public repository that
 383                 # early in its history moved trunk => public/trunk we would miss that
 384                 # earlier history without allowing the funky shorten+prefix behavior.
 385                 # So we read back the svn-remote.svn.fetch configuration and compute
 386                 # the prefix.  This way we are sure to get the correct prefix.
 387                 gitsvnurl="$(git config --get svn-remote.svn.url)" || :
 388                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1)" || :
 389                 gitsvnprefix="${gitsvnfetch%%:*}"
 390                 gitsvnsuffix="${gitsvnprefix##*/}"
 391                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 392                 # Ask git-svn to store everything in the normal non-remote
 393                 # locations being careful to use the correct prefix
 394                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 395                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 396                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 397                 # look for additional non-standard directories to fetch
 398                 # check for standard layout at the same time
 399                 foundstd=
 400                 foundfile=
 401                 svn --non-interactive --username "$svnuser" --password "$svnpass" ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null |
 402                 { while read file; do case $file in
 403                         # skip the already-handled standard ones and any with a space or tab
 404                         *'      '*|*' '*) :;;
 405                         trunk/|branches/|tags/) foundstd=1;;
 406                         # only fetch extra directories from the $svnurl root (not any files)
 407                         *?/) git config --add svn-remote.svn.fetch \
 408                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 409                         *?) foundfile=1;;
 410                 esac; done
 411                 # if files found and no standard directories present use a simpler layout
 412                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 413                         git config --unset svn-remote.svn.branches
 414                         git config --unset svn-remote.svn.tags
 415                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 416                 fi; }
 417                 test $? -eq 0
 418                 # git svn fetch on a very large repo can take some time and the
 419                 # remote server may interrupt the connection from time to time.
 420                 # keep retrying (after a brief pause) as long as we are making progress.
 421                 # however, we do limit the total number of retries to 1000
 422                 # we will, however, retry up to 5 times even if we're not making progress
 423                 v_get_svn_progress_fingerprint() {
 424                         eval "$1="'"$({ GIT_DIR=. git svn info <"$mtlinesfile" 2>&1; git show-ref --head 2>&1; } |
 425                         git hash-object -t blob --stdin )"' || :
 426                 }
 427                 svn_ret_err() { return "${1:-1}"; }
 428                 svn_retries=1000 # maximum possible fetch attempts no matter what
 429                 svn_retry_backoff_start_half=60 # min retry wait is double this amount in seconds
 430                 svn_backoff_count=7 # max retry wait is $svn_retry_backoff_start_half * 2^$svn_backoff_count
 431                 # Cumulative backoff wait before giving up on consecutive no-progress retries
 432                 #   is approximately 2 * $svn_retry_backoff_start_half * 2^$svn_backoff_count
 433                 # For a $svn_backoff_count of 7 that works out to be exactly 4h14m
 434                 svn_progress=
 435                 v_get_svn_progress_fingerprint svn_progress
 436                 svn_progress_retries="$svn_retries"
 437                 svn_retry_backoff="$svn_retry_backoff_start_half"
 438                 svn_err=0
 439                 while [ "$svn_retries" -gt 0 ]; do
 440                         svn_retries="$(( $svn_retries - 1 ))"
 441                         svn_err=0
 442                         GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1
 443                         export GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 444                         unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 445                         saveconfig="$GIT_CONFIG_PARAMETERS"
 446                         git_add_config 'gc.auto=1'
 447                         git_add_config 'gc.autoPackLimit=1'
 448                         # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 449                         GIT_DIR=. git_ulimit svn fetch --log-window-size=$var_log_window_size --username="$svnuser" --quiet <"$mtlinesfile" || svn_err="$?"
 450                         GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 451                         export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 452                         unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 453                         unset GIT_CONFIG_PARAMETERS
 454                         [ -z "$saveconfig" ] || {
 455                                 GIT_CONFIG_PARAMETERS="$saveconfig"
 456                                 export GIT_CONFIG_PARAMETERS
 457                         }
 458                         [ "${svn_err:-1}" -ne 0 ] || break # success!
 459                         # Check to see if we made any progress
 460                         v_get_svn_progress_fingerprint svn_progress_now
 461                         if [ "$svn_progress_now" != "$svn_progress" ]; then
 462                                 # we made progress, continue the loop with min wait
 463                                 svn_progress="$svn_progress_now"
 464                                 svn_progress_retries="$svn_retries"
 465                                 svn_retry_backoff="$svn_retry_backoff_start_half"
 466                         else
 467                                 # no progress, but we only give up after
 468                                 # $svn_backoff_count no-progress attempts in a row
 469                                 [ "$(( $svn_progress_retries - $svn_retries ))" -lt "$svn_backoff_count" ] ||
 470                                         break # failure
 471                                 # continue but only after twice the previous wait
 472                                 # (which will still be the min wait if this is the
 473                                 # first no-progress retry after making some progress)
 474                         fi
 475                         svn_retry_backoff="$(( 2 * $svn_retry_backoff ))"
 476                         # Pause for $svn_retry_backoff seconds before retrying to be friendly to the server
 477                         # Use that time to pack up loose objects if there are "lotsa" them
 478                         if ! lotsa_loose_objects_or_sopacks; then
 479                                 echo "Pausing for $svn_retry_backoff seconds before retrying ($(date))"
 480                                 sleep "$svn_retry_backoff"
 481                         else
 482                                 pausestop="$(( $(date '+%s') + $svn_retry_backoff ))"
 483                                 echo "Pausing and packing loose objects for $svn_retry_backoff seconds before retrying ($(date))"
 484                                 pack_incremental_loose_objects_if_lockable ||
 485                                 echo "Packing skipped (only pausing): $lockerr"
 486                                 timenow="$(date '+%s')"
 487                                 if [ "$timenow" -lt "$pausestop" ]; then
 488                                         sleepamt="$(( $pausestop - $timenow ))"
 489                                         [ "$sleepamt" -le "$svn_retry_backoff" ] ||
 490                                                 sleepamt="$svn_retry_backoff" # paranoia check
 491                                         sleep "$sleepamt"
 492                                 fi
 493                         fi
 494                         cleanup_git_svn_leftovers
 495                         echo "Retrying fetch ($(date))"
 496                 done
 497                 [ "${svn_err:-1}" -eq 0 ] || svn_ret_err "$svn_err"
 498                 test ${svn_err:-1} -eq 0
 499                 # git svn does not preserve group permissions in the svn subdirectory
 500                 chmod -R ug+rw,o+r svn
 501                 # git svn also leaves behind ref turds that end with @nnn
 502                 # We get rid of them now
 503                 git for-each-ref --format='%(refname)' |
 504                 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
 505                 git_updateref_stdin
 506                 unset GIT_ASKPASS_PASSWORD
 507                 ;;
 508         darcs://* | darcs+http://* | darcs+https://*)
 509                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 510                 case "$url" in
 511                         darcs://*) darcsurl="http://${url#darcs://}";;
 512                         *) darcsurl="${url#darcs+}";;
 513                 esac
 514                 git_darcs_fetch "$darcsurl"
 515                 ;;
 516         bzr://*)
 517                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 518                 # we just remove bzr:// here, a typical bzr url is just
 519                 # "lp:foo"
 520                 bzrurl="${url#bzr://}"
 521                 git_bzr_fetch "$bzrurl"
 522                 ;;
 523         hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
 524                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 525                 # We just remove hg+ here, so hg+http://... becomes http://...
 526                 hgurl="${url#hg+}"
 527                 # Perform the initial hg clone
 528                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 529                 # Do the fast-export | fast-import
 530                 git_hg_fetch
 531                 ;;
 532         *)
 533                 # We manually add remote.origin.url and remote.origin.fetch
 534                 # to simulate a `git remote add --mirror=fetch` since that's
 535                 # not available until Git 1.7.5 and this way we guarantee we
 536                 # always get exactly the intended configuration and nothing else.
 537                 git config remote.origin.url "$url"
 538                 if ! is_gfi_mirror_url "$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
 539                         git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
 540                         git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
 541                         git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
 542                         git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
 543                         git config --bool girocco.lastupdateclean true
 544                 else
 545                         git config --replace-all remote.origin.fetch "+refs/*:refs/*"
 546                         git config --bool girocco.lastupdateclean false
 547                 fi
 548                 # Set the correct HEAD symref by using ls-remote first
 549                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp ||
 550                 {
 551                         # Since everything was redirected, on failure there'd be no output,
 552                         # so let's make some failure output
 553                         cat .pkts-temp
 554                         echo ""
 555                         echo "git ls-remote \"$url\" failed"
 556                         exit 1
 557                 }
 558                 # Compensate for git() {} side effects
 559                 unset GIT_TRACE_PACKET
 560                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 561                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 562                 # out of the packet trace data.
 563                 if [ -s .refs-temp ]; then
 564                         # Nothing to do unless the remote repository has at least 1 ref
 565                         # See if we got a HEAD ref
 566                         head="$(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+HEAD\$" <.refs-temp | LC_ALL=C awk '{print $1}')"
 567                         # If the remote has HEAD set to a symbolic ref that does not exist
 568                         # then we will not receive a HEAD ref in the ls-remote output
 569                         headref=
 570                         showheadwarn=
 571                         symrefcap=
 572                         if [ -n "$head" ]; then
 573                                 symrefcap="$(LC_ALL=C sed -ne <.pkts-temp \
 574                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 575                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 576                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 577                                 # matches HEAD otherwise take the first refs/heads/... match
 578                                 matchcnt=0
 579                                 while read ref; do
 580                                         [ -n "$ref" ] || continue
 581                                         matchcnt=$(( $matchcnt + 1 ))
 582                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 583                                                 headref="$ref"
 584                                         fi
 585                                         if [ "$headref" = "${symrefcap:-refs/heads/master}" ] && [ $matchcnt -gt 1 ]; then
 586                                                 break
 587                                         fi
 588                                 done <<-EOT
 589                                 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 590                                         LC_ALL=C awk '{print $2}')
 591                                 EOT
 592                                 # Warn if there was more than one match and $symrefcap is empty
 593                                 # or $symrefcap is not the same as $headref since our choice might
 594                                 # differ from the source repository's HEAD
 595                                 if [ $matchcnt -ge 1 ] && [ "$symrefcap" != "$headref" ] &&
 596                                    { [ -n "$symrefcap" ] || [ $matchcnt -gt 1 ]; }; then
 597                                         showheadwarn=1
 598                                 fi
 599                         fi
 600                         if [ -z "$headref" ]; then
 601                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 602                                 # if it exists otherwise take the first refs/heads/...
 603                                 # We do not support having a detached HEAD.
 604                                 # We always warn now because we will be setting HEAD differently
 605                                 # than the source repository had HEAD set
 606                                 showheadwarn=1
 607                                 while read ref; do
 608                                         [ -n "$ref" ] || continue
 609                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 610                                                 headref="$ref"
 611                                         fi
 612                                         [ "$headref" != "refs/heads/master" ] || break
 613                                 done <<-EOT
 614                                 $(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 615                                         LC_ALL=C awk '{print $2}')
 616                                 EOT
 617                         fi
 618                         # If we STILL do not have a HEAD ref (perhaps the source repository
 619                         # contains only tags) then use refs/heads/master.  It will be invalid
 620                         # but is no worse than we used to do by default and we'll warn about
 621                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 622                         [ -n "$headref" ] || headref="refs/heads/master"
 623                         git symbolic-ref HEAD "$headref"
 624                         pruneopt=--prune
 625                         [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
 626                         # remember the starting time so we can easily detect new packs for fast-import mirrors
 627                         # we sleep for 1 second after creating .gfipack to make sure all packs are newer
 628                         if is_gfi_mirror_url "$url" && ! [ -e .gfipack ]; then
 629                                 rm -f .gfipack
 630                                 >.gfipack
 631                                 sleep 1
 632                         fi
 633                         GIT_SSL_NO_VERIFY=1 git_ulimit remote update $pruneopt
 634                         if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
 635                                 find -L objects/pack -type f -newer .gfipack -name "pack-$octet20*.pack" -print >>gfi-packs
 636                                 rm -f .gfipack
 637                         fi
 638                 else
 639                         warnempty=1
 640                         git symbolic-ref HEAD "refs/heads/master"
 641                 fi
 642                 rm -f .refs-temp .pkts-temp
 643                 ;;
 644 esac
 645
 646 # Check the max_clone_objects setting now (if set)
 647 if [ "${cfg_max_clone_objects:-0}" != "0" ]; then
 648         objcount="$(git count-objects -v | LC_ALL=C awk 'BEGIN{v=0}/^count:/||/^in-pack:/{v+=$2}END{print v}')" || :
 649         if [ -n "$objcount" ] && [ "$objcount" -gt "$cfg_max_clone_objects" ]; then
 650                 exit_objs="$objcount"
 651                 exit 1 # fail the clone
 652         fi
 653 fi
 654
 655 # The objects subdirectories permissions must be updated now.
 656 # In the case of a dumb http clone, the permissions will not be correct
 657 # (missing group write) despite the core.sharedrepository=1 setting!
 658 # The objects themselves seem to have the correct permissions.
 659 # This problem appears to have been fixed in the most recent git versions.
 660 perms=g+w
 661 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 662 chmod $perms $(find -L objects -maxdepth 1 -type d) 2>/dev/null || :
 663
 664 # We may have just cloned a lot of refs and they will all be
 665 # individual files at this point.  Let's pack them now so we
 666 # can have better performance right from the start.
 667 git pack-refs --all
 668
 669 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 670 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 671 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 672 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 673         --count=1 refs/heads >info/lastactivity || :
 674 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 675
 676 # Don't leave a multi-megabyte useless FETCH_HEAD behind
 677 rm -f FETCH_HEAD
 678
 679 # Last ditch attempt to get a valid HEAD for a non-git source
 680 check_and_set_head || :
 681
 682 # The rest
 683 echo "Final touches..."
 684 git update-server-info
 685 trap "" EXIT
 686
 687 # run gc now unless the clone is empty
 688 if [ -z "$warnempty" ]; then
 689         git config --unset gitweb.lastgc 2>/dev/null || :
 690         rm -f .delaygc .allowgc
 691 fi
 692
 693 emptynote=
 694 [ -z "$warnempty" ] ||
 695 emptynote="
 696 WARNING:  You have mirrored an empty repository.
 697 "
 698 headnote=
 699 [ -z "$showheadwarn" ] || [ -z "$headref" ] ||
 700 headnote="
 701 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 702        Use the \"Project settings\" link to choose a different HEAD symref.
 703 "
 704 sizenote=
 705 ! is_gfi_mirror ||
 706 sizenote="
 707 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 708        size may be somewhat larger than necessary.  This will be corrected
 709        shortly.  If you intend to clone this repository you may want to
 710        wait up to 1 hour before doing so in order to receive the more
 711        compact final size.
 712 "
 713 [ -z "$mailaddrs" ] ||
 714 mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 715 Congratulations! The clone of project $proj just completed.
 716
 717         * Source URL: $url
 718         * GitWeb interface: $cfg_gitweburl/$proj.git
 719         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 720 $emptynote$headnote$sizenote
 721 Have a lot of fun.
 722 EOT
 723
 724 echo "Mirroring finished successfuly!"
 725 # In case this is a re-mirror, lastgc could have been set already so clear it now
 726 git config --unset gitweb.lastgc || :
 727 rm .clone_in_progress
 728 echo "$sizenote@OVER@"