taskd/clone.sh

   1 #!/bin/sh
   2 #
   3 # Invoked from taskd/taskd.pl
   4
   5 . @basedir@/shlib.sh
   6 . @basedir@/jobd/gc-util-functions.sh
   7
   8 set -e
   9
  10 umask 002
  11 [ "$cfg_permission_control" != "Hooks" ] || umask 000
  12 clean_git_env
  13
  14 # darcs fast-export | git fast-import with error handling
  15 git_darcs_fetch() (
  16         set_utf8_locale
  17         _err1=
  18         _err2=
  19         exec 3>&1
  20         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  21         $(
  22                 exec 4>&3 3>&1 1>&4 4>&-
  23                 {
  24                         _e1=0
  25                         "$cfg_basedir"/bin/darcs-fast-export \
  26                                 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  27                         echo $_e1 >&3
  28                 } |
  29                 {
  30                         _e2=0
  31                         git_ulimit fast-import \
  32                                 --export-marks="$(pwd)/gfi-marks" \
  33                                 --export-pack-edges="$(pwd)/gfi-packs" \
  34                                 --force 3>&- || _e2=$?
  35                         echo $_e2 >&3
  36                 }
  37         )
  38         EOT
  39         exec 3>&-
  40         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  41         return $?
  42 )
  43
  44 # bzr fast-export | git fast-import with error handling
  45 git_bzr_fetch() (
  46         set_utf8_locale
  47         BZR_LOG=/dev/null
  48         export BZR_LOG
  49         _err1=
  50         _err2=
  51         exec 3>&1
  52         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  53         $(
  54                 exec 4>&3 3>&1 1>&4 4>&-
  55                 {
  56                         _e1=0
  57                         bzr fast-export --plain \
  58                                 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  59                         echo $_e1 >&3
  60                 } |
  61                 {
  62                         _e2=0
  63                         git_ulimit fast-import \
  64                                 --export-marks="$(pwd)/gfi-marks" \
  65                                 --export-pack-edges="$(pwd)/gfi-packs" \
  66                                 --force 3>&- || _e2=$?
  67                         echo $_e2 >&3
  68                 }
  69         )
  70         EOT
  71         exec 3>&-
  72         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  73         return $?
  74 )
  75
  76 clear_all_objects_and_packs() {
  77         if [ -d objects ]; then
  78                 # make sure the repository is not left broken
  79                 printf '%s\n' 'ref: refs/heads/master' >HEAD || :
  80                 rm -f packed-refs || :
  81                 find -H refs objects -type f -exec rm -f '{}' + >/dev/null 2>&1 || :
  82                 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
  83         fi
  84 }
  85
  86 exit_err=0
  87 exit_objs=0
  88 send_clone_failed() {
  89         trap "" EXIT
  90         # We must now close the .clonelog file that is open on stdout and stderr
  91         exec >/dev/null 2>&1
  92         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
  93         failaddrs="$(config_get owner)" || :
  94         ccadm="${cfg_admincc:-0}"
  95         xfsz_err=""
  96         if [ "${exit_err:-0}" = "${var_xfsz_err:-999}" ]; then
  97                 ccadm=1
  98                 reposize="$(cd objects && du -sk . | LC_ALL=C awk '{print $1}')" || :
  99                 if [ -n "$reposize" ]; then
 100                         if [ $reposize -lt 5120 ]; then
 101                                 reposize="$reposize KiB"
 102                         else
 103                                 reposize="$(( $reposize / 1024 ))"
 104                                 if [ $reposize -lt 5120 ]; then
 105                                         reposize="$reposize MiB"
 106                                 else
 107                                         reposize="$(( $reposize / 1024 ))"
 108                                         reposize="$reposize GiB"
 109                                 fi
 110                         fi
 111                 fi
 112                 xfsz_err="
 113
 114 The source repository${reposize:+ ($reposize)} exceeds our maximum allowed repository size."
 115                 clear_all_objects_and_packs
 116         fi
 117         xobjs_err=""
 118         if [ "${exit_objs:-0}" != "0" ]; then
 119                 ccadm=1
 120                 xobjs_err="
 121
 122 The source repository${exit_objs:+ ($exit_objs objects)} exceeds our maximum allowed object limit."
 123                 clear_all_objects_and_packs
 124         fi
 125         [ "$ccadm" = "0" ] || [ -z "$cfg_admin" ] ||
 126         if [ -z "$failaddrs" ]; then failaddrs="$cfg_admin"; else failaddrs="$failaddrs,$cfg_admin"; fi
 127         [ -z "$failaddrs" ] ||
 128         {
 129               cat <<EOT
 130 Condolences. The clone of project $proj just failed.$xfsz_err$xobjs_err
 131
 132         * Source URL: $url
 133         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 134
 135 The project settings link may be used to adjust the settings
 136 and restart the clone in order to try the clone again.
 137 EOT
 138                 if [ -f .clonelog ] && [ -r .clonelog ]; then
 139                         echo ""
 140                         echo "Log follows:"
 141                         echo ""
 142                         loglines=$(LC_ALL=C wc -l <.clonelog)
 143                         if [ $loglines -le 203 ]; then
 144                                 cat .clonelog
 145                         else
 146                                 head -n 100 .clonelog
 147                                 echo ""
 148                                 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
 149                                 echo ""
 150                                 tail -n 100 .clonelog
 151                         fi
 152                 fi
 153         } | mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone failed" "$failaddrs" || :
 154 }
 155
 156 # removes any git-svn leftovers
 157 cleanup_git_svn_leftovers() {
 158
 159         # Remove any stale git-svn temp files
 160         # The git-svn process creates temp files with random 10 character names
 161         # in the root of $GIT_DIR.  Unfortunately they do not have a recognizable
 162         # prefix, so we just have to kill any files with a 10-character name.
 163         # All characters are chosen from
 164         # [A-Za-z0-9_] so we can at least check that and fortunately the only
 165         # collision is 'FETCH_HEAD' but that doesn't matter.
 166         # There may also be temp files with a Git_ prefix as well.
 167         _randchar='[A-Za-z0-9_]'
 168         _randchar2="$_randchar$_randchar"
 169         _randchar4="$_randchar2$_randchar2"
 170         _randchar10="$_randchar4$_randchar4$_randchar2"
 171         find -L . -maxdepth 1 -type f -name "$_randchar10" -exec rm -f '{}' + || :
 172         find -L . -maxdepth 1 -type f -name "Git_*" -exec rm -f '{}' + || :
 173 }
 174
 175 # removes all leftovers from a previous failed clone attempt
 176 cleanup_failed_clone() {
 177
 178         # Remove any left-over svn-remote.svn or remote.origin config
 179         git config --remove-section svn-remote.svn 2>/dev/null || :
 180         git config --remove-section remote.origin 2>/dev/null || :
 181
 182         # If there is a remote-template.origin section, pre-seed the
 183         # remote.origin section with its contents
 184         git config --get-regexp '^remote-template\.origin\..' |
 185         while read name value; do
 186                 if [ -n "$name" ] && [ -n "$value" ]; then
 187                         git config "remote${name#remote-template}" "$value"
 188                 fi
 189         done
 190
 191         # Any pre-existing FETCH_HEAD from a previous clone failed or not is
 192         # now garbage to be removed
 193         rm -f FETCH_HEAD
 194
 195         # Remove any stale ref locks
 196         clear_stale_ref_locks
 197
 198         # Remove any left-over svn dir from a previous failed attempt
 199         rm -rf svn
 200
 201         # Remove any left-over .darcs dirs from a previous failed attempt
 202         rm -rf *.darcs
 203
 204         # Remove any left-over repo.hg dir from a previous failed attempt
 205         rm -rf repo.hg
 206
 207         # Remove any left-over import/export/temp files from a previous failed attempt
 208         rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks* hg2git-state \
 209                 gfi-marks gfi-packs .pkts-temp .refs-temp
 210
 211         # Remove any git-svn junk
 212         cleanup_git_svn_leftovers
 213
 214         # We want a gc right after the clone, so re-enable that just in case.
 215         # There's a potential race where we could add it and gc.sh could remove
 216         # it, but we'll reunset lastgc just before we remove .delaygc at the end.
 217         [ -e .delaygc ] || >.delaygc
 218         git config --unset gitweb.lastgc 2>/dev/null || :
 219
 220         # Remove all pre-existing refs
 221         rm -f packed-refs
 222         git for-each-ref --format='delete %(refname)' | git_updateref_stdin 2>/dev/null || :
 223
 224         # The initial state before a clone starts has HEAD as a symbolic-ref to master
 225         git symbolic-ref HEAD refs/heads/master
 226
 227         # HEAD is no longer "ok"
 228         git config --unset girocco.headok 2>/dev/null || :
 229
 230         # We, perhaps, ought to remove any packs/loose objects now, but the next gc
 231         # will get rid of any extras.  Also, if we're recloning the same thing, any
 232         # preexisting packs/loose objects containing what we're recloning will only
 233         # speed up the reclone by avoiding some disk writes.  So we don't kill them.
 234
 235         # It's just remotely possible that a bunch of failures in a row could
 236         # create a big mess that just keeps growing and growing...
 237         # Trigger a .needsgc if that happens.
 238         check_and_set_needsgc
 239 }
 240
 241 proj="${1%.git}"
 242 cd "$cfg_reporoot/$proj.git"
 243 bang_reset
 244
 245 ! [ -e .delaygc ] || >.allowgc || :
 246
 247 trap "exit_err=$?; echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
 248 echo "Project: $proj"
 249 echo "   Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
 250 echo ""
 251 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; }
 252 url="$(config_get baseurl)" || :
 253 case "$url" in *"       "*|*" "*|"")
 254         echo "Bad mirror URL (\"$url\")"
 255         exit 1
 256 esac
 257
 258 cleanup_failed_clone
 259
 260 # Record original mirror type for use by update.sh
 261 mirror_type="$(get_url_mirror_type "$url")"
 262 git config girocco.mirrortype "$mirror_type"
 263
 264 echo "Mirroring from URL \"$url\""
 265 echo ""
 266
 267 if [ "$cfg_project_owners" = "source" ]; then
 268         config set owner "$(ls -ldH "${url#file://}" 2>/dev/null | LC_ALL=C awk '{print $3}')"
 269 fi
 270
 271 mailaddrs="$(config_get owner)" || :
 272 [ -z "$cfg_admin" ] ||
 273 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 274
 275 # Make sure we don't get any unwanted loose objects
 276 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
 277 # tweak its configuration to prevent that
 278 git_add_config 'fetch.unpackLimit=1'
 279 # Note the git config documentation is wrong
 280 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
 281 git_add_config 'transfer.unpackLimit=1'
 282 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
 283 git_add_config 'fastimport.unpackLimit=0'
 284
 285 # Initial mirror
 286 echo "Initiating mirroring..."
 287 headref=
 288 showheadwarn=
 289 warnempty=
 290
 291 # remember the starting time so we can easily combine fetched loose objects
 292 # we sleep for 1 second after creating .needspack to make sure all objects are newer
 293 if ! [ -e .needspack ]; then
 294         rm -f .needspack
 295         >.needspack
 296         sleep 1
 297 fi
 298
 299 case "$url" in
 300         svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 301                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; }
 302                 # Allow the username to be specified in the "svn-credential.svn.username"
 303                 # property and the password in the "svn-credential.svn.password" property
 304                 # Use an 'anonsvn' username by default as is commonly used for anonymous svn
 305                 # Default the password to the same as the username
 306                 # The password property will be ignored unless a username has been specified
 307                 if svnuser="$(git config --get svn-credential.svn.username)" && [ -n "$svnuser" ]; then
 308                         if ! svnpass="$(git config --get svn-credential.svn.password)"; then
 309                                 svnpass="$svnuser"
 310                         fi
 311                         url1="${url#*://}"
 312                         url1="${url1%%/*}"
 313                         case "$url1" in ?*"@"?*)
 314                                 urlsch="${url%%://*}"
 315                                 url="$urlsch://${url#*@}"
 316                         esac
 317                 else
 318                         # As a fallback, check in the URL, just in case
 319                         url1="${url#*://}"
 320                         url1="${url1%%/*}"
 321                         svnuser=
 322                         case "$url1" in ?*"@"?*)
 323                                 urlsch="${url%%://*}"
 324                                 url="$urlsch://${url#*@}"
 325                                 url1="${url1%%@*}"
 326                                 svnuser="${url1%%:*}"
 327                                 if [ -n "$svnuser" ]; then
 328                                         svnpass="$svnuser"
 329                                         case "$url1" in *":"*)
 330                                                 svnpass="${url1#*:}"
 331                                         esac
 332                                 fi
 333                         esac
 334                         if [ -z "$svnuser" ]; then
 335                                 svnuser="anonsvn"
 336                                 svnpass="anonsvn"
 337                         fi
 338                 fi
 339                 GIT_ASKPASS_PASSWORD="$svnpass"
 340                 export GIT_ASKPASS_PASSWORD
 341                 # We just remove svn+ here, so svn+http://... becomes http://...
 342                 # We also remove a trailing '/' to match what git-svn will do
 343                 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
 344                 svnurl="${svnurl%/}"
 345                 # We require svn info to succeed on the URL otherwise it's
 346                 # simply not a valid URL and without using -s on the init it
 347                 # will not otherwise be tested until the fetch
 348                 svn --non-interactive --username "$svnuser" --password "$svnpass" info "$svnurl" >/dev/null
 349                 # We initially use -s for the init which will possibly shorten
 350                 # the URL.  However, the shortening can fail if a password is
 351                 # not required for the longer version but is for the shorter,
 352                 # so try again without -s if the -s version fails.
 353                 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
 354                 # create a .git subdirectory!
 355                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" -s "$svnurl" <"$mtlinesfile" ||
 356                 GIT_DIR=. git svn init --username="$svnuser" --prefix "" "$svnurl" <"$mtlinesfile"
 357                 # We need to remember this url so we can detect changes because
 358                 # ever so "helpful" git-svn may shorten it!
 359                 config_set svnurl "$svnurl"
 360                 # At this point, since we asked for a standard layout (-s) git-svn
 361                 # may have been "helpful" and adjusted our $svnurl to a prefix and
 362                 # then glued the removed suffix onto the front of any svn-remote.svn.*
 363                 # config items.  We could avoid this by not using the '-s' option
 364                 # but then we might not get all the history.  If, for example, we
 365                 # are cloning an http://svn.example.com/repos/public repository that
 366                 # early in its history moved trunk => public/trunk we would miss that
 367                 # earlier history without allowing the funky shorten+prefix behavior.
 368                 # So we read back the svn-remote.svn.fetch configuration and compute
 369                 # the prefix.  This way we are sure to get the correct prefix.
 370                 gitsvnurl="$(git config --get svn-remote.svn.url)" || :
 371                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1)" || :
 372                 gitsvnprefix="${gitsvnfetch%%:*}"
 373                 gitsvnsuffix="${gitsvnprefix##*/}"
 374                 gitsvnprefix="${gitsvnprefix%$gitsvnsuffix}"
 375                 # Ask git-svn to store everything in the normal non-remote
 376                 # locations being careful to use the correct prefix
 377                 git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master"
 378                 git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*"
 379                 git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*"
 380                 # look for additional non-standard directories to fetch
 381                 # check for standard layout at the same time
 382                 foundstd=
 383                 foundfile=
 384                 svn --non-interactive --username "$svnuser" --password "$svnpass" ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null |
 385                 { while read file; do case $file in
 386                         # skip the already-handled standard ones and any with a space or tab
 387                         *'      '*|*' '*) :;;
 388                         trunk/|branches/|tags/) foundstd=1;;
 389                         # only fetch extra directories from the $svnurl root (not any files)
 390                         *?/) git config --add svn-remote.svn.fetch \
 391                                 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
 392                         *?) foundfile=1;;
 393                 esac; done
 394                 # if files found and no standard directories present use a simpler layout
 395                 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
 396                         git config --unset svn-remote.svn.branches
 397                         git config --unset svn-remote.svn.tags
 398                         git config --replace-all svn-remote.svn.fetch ':refs/heads/master'
 399                 fi; }
 400                 test $? -eq 0
 401                 # git svn fetch on a very large repo can take some time and the
 402                 # remote server may interrupt the connection from time to time.
 403                 # keep retrying (after a brief pause) as long as we are making progress.
 404                 # however, we do limit the total number of retries to 1000
 405                 # we will, however, retry up to 5 times even if we're not making progress
 406                 v_get_svn_progress_fingerprint() {
 407                         eval "$1="'"$({ GIT_DIR=. git svn info <"$mtlinesfile" 2>&1; git show-ref --head 2>&1; } |
 408                         git hash-object -t blob --stdin )"' || :
 409                 }
 410                 svn_ret_err() { return "${1:-1}"; }
 411                 svn_retries=1000 # maximum possible fetch attempts no matter what
 412                 svn_retry_backoff_start_half=60 # min retry wait is double this amount in seconds
 413                 svn_backoff_count=7 # max retry wait is $svn_retry_backoff_start_half * 2^$svn_backoff_count
 414                 # Cumulative backoff wait before giving up on consecutive no-progress retries
 415                 #   is approximately 2 * $svn_retry_backoff_start_half * 2^$svn_backoff_count
 416                 # For a $svn_backoff_count of 7 that works out to be exactly 4h14m
 417                 svn_progress=
 418                 v_get_svn_progress_fingerprint svn_progress
 419                 svn_progress_retries="$svn_retries"
 420                 svn_retry_backoff="$svn_retry_backoff_start_half"
 421                 svn_err=0
 422                 while [ "$svn_retries" -gt 0 ]; do
 423                         svn_retries="$(( $svn_retries - 1 ))"
 424                         svn_err=0
 425                         GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1
 426                         export GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 427                         unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 428                         saveconfig="$GIT_CONFIG_PARAMETERS"
 429                         git_add_config 'gc.auto=1'
 430                         git_add_config 'gc.autoPackLimit=1'
 431                         # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
 432                         GIT_DIR=. git_ulimit svn fetch --log-window-size=$var_log_window_size --username="$svnuser" --quiet <"$mtlinesfile" || svn_err="$?"
 433                         GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 434                         export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 435                         unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 436                         unset GIT_CONFIG_PARAMETERS
 437                         [ -z "$saveconfig" ] || {
 438                                 GIT_CONFIG_PARAMETERS="$saveconfig"
 439                                 export GIT_CONFIG_PARAMETERS
 440                         }
 441                         [ "${svn_err:-1}" -ne 0 ] || break # success!
 442                         # Check to see if we made any progress
 443                         v_get_svn_progress_fingerprint svn_progress_now
 444                         if [ "$svn_progress_now" != "$svn_progress" ]; then
 445                                 # we made progress, continue the loop with min wait
 446                                 svn_progress="$svn_progress_now"
 447                                 svn_progress_retries="$svn_retries"
 448                                 svn_retry_backoff="$svn_retry_backoff_start_half"
 449                         else
 450                                 # no progress, but we only give up after
 451                                 # $svn_backoff_count no-progress attempts in a row
 452                                 [ "$(( $svn_progress_retries - $svn_retries ))" -lt "$svn_backoff_count" ] ||
 453                                         break # failure
 454                                 # continue but only after twice the previous wait
 455                                 # (which will still be the min wait if this is the
 456                                 # first no-progress retry after making some progress)
 457                         fi
 458                         svn_retry_backoff="$(( 2 * $svn_retry_backoff ))"
 459                         # Pause for $svn_retry_backoff seconds before retrying to be friendly to the server
 460                         # Use that time to pack up loose objects if there are "lotsa" them
 461                         if ! lotsa_loose_objects_or_sopacks; then
 462                                 echo "Pausing for $svn_retry_backoff seconds before retrying ($(date))"
 463                                 sleep "$svn_retry_backoff"
 464                         else
 465                                 pausestop="$(( $(date '+%s') + $svn_retry_backoff ))"
 466                                 echo "Pausing and packing loose objects for $svn_retry_backoff seconds before retrying ($(date))"
 467                                 pack_incremental_loose_objects_if_lockable ||
 468                                 echo "Packing skipped (only pausing): $lockerr"
 469                                 timenow="$(date '+%s')"
 470                                 if [ "$timenow" -lt "$pausestop" ]; then
 471                                         sleepamt="$(( $pausestop - $timenow ))"
 472                                         [ "$sleepamt" -le "$svn_retry_backoff" ] ||
 473                                                 sleepamt="$svn_retry_backoff" # paranoia check
 474                                         sleep "$sleepamt"
 475                                 fi
 476                         fi
 477                         cleanup_git_svn_leftovers
 478                         echo "Retrying fetch ($(date))"
 479                 done
 480                 [ "${svn_err:-1}" -eq 0 ] || svn_ret_err "$svn_err"
 481                 test ${svn_err:-1} -eq 0
 482                 # git svn does not preserve group permissions in the svn subdirectory
 483                 chmod -R ug+rw,o+r svn
 484                 # git svn also leaves behind ref turds that end with @nnn
 485                 # We get rid of them now
 486                 git for-each-ref --format='%(refname)' |
 487                 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
 488                 git_updateref_stdin
 489                 unset GIT_ASKPASS_PASSWORD
 490                 ;;
 491         darcs://* | darcs+http://* | darcs+https://*)
 492                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; }
 493                 case "$url" in
 494                         darcs://*) darcsurl="http://${url#darcs://}";;
 495                         *) darcsurl="${url#darcs+}";;
 496                 esac
 497                 git_darcs_fetch "$darcsurl"
 498                 ;;
 499         bzr://*)
 500                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; }
 501                 # we just remove bzr:// here, a typical bzr url is just
 502                 # "lp:foo"
 503                 bzrurl="${url#bzr://}"
 504                 git_bzr_fetch "$bzrurl"
 505                 ;;
 506         hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
 507                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
 508                 # We just remove hg+ here, so hg+http://... becomes http://...
 509                 hgurl="${url#hg+}"
 510                 # Perform the initial hg clone
 511                 hg clone -U "$hgurl" "$(pwd)/repo.hg"
 512                 # Do the fast-export | fast-import
 513                 git_hg_fetch
 514                 ;;
 515         *)
 516                 # We manually add remote.origin.url and remote.origin.fetch
 517                 # to simulate a `git remote add --mirror=fetch` since that's
 518                 # not available until Git 1.7.5 and this way we guarantee we
 519                 # always get exactly the intended configuration and nothing else.
 520                 git config remote.origin.url "$url"
 521                 if ! is_gfi_mirror_url "$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
 522                         git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
 523                         git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
 524                         git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
 525                         git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
 526                         git config --bool girocco.lastupdateclean true
 527                 else
 528                         git config --replace-all remote.origin.fetch "+refs/*:refs/*"
 529                         git config --bool girocco.lastupdateclean false
 530                 fi
 531                 # Set the correct HEAD symref by using ls-remote first
 532                 GIT_SSL_NO_VERIFY=1 GIT_TRACE_PACKET=1 git ls-remote origin >.refs-temp 2>.pkts-temp ||
 533                 {
 534                         # Since everything was redirected, on failure there'd be no output,
 535                         # so let's make some failure output
 536                         cat .pkts-temp
 537                         echo ""
 538                         echo "git ls-remote \"$url\" failed"
 539                         exit 1
 540                 }
 541                 # Compensate for git() {} side effects
 542                 unset GIT_TRACE_PACKET
 543                 # If the server is running at least Git 1.8.4.3 then it will send us the actual
 544                 # symref for HEAD.  If we are running at least Git 1.7.5 then we can snarf that
 545                 # out of the packet trace data.
 546                 if [ -s .refs-temp ]; then
 547                         # Nothing to do unless the remote repository has at least 1 ref
 548                         # See if we got a HEAD ref
 549                         head="$(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+HEAD\$" <.refs-temp | LC_ALL=C awk '{print $1}')"
 550                         # If the remote has HEAD set to a symbolic ref that does not exist
 551                         # then we will not receive a HEAD ref in the ls-remote output
 552                         headref=
 553                         showheadwarn=
 554                         symrefcap=
 555                         if [ -n "$head" ]; then
 556                                 symrefcap="$(LC_ALL=C sed -ne <.pkts-temp \
 557                                         "/packet:.*git<.*[ $tab]symref="'HEAD:refs\/heads\/'"[^ $tab]/\
 558                                         {s/^.*[ $tab]symref="'HEAD:\(refs\/heads\/'"[^ $tab][^ $tab]*"'\).*$/\1/;p;}')"
 559                                 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
 560                                 # matches HEAD otherwise take the first refs/heads/... match
 561                                 matchcnt=0
 562                                 while read ref; do
 563                                         [ -n "$ref" ] || continue
 564                                         matchcnt=$(( $matchcnt + 1 ))
 565                                         if [ -z "$headref" ] || [ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
 566                                                 headref="$ref"
 567                                         fi
 568                                         if [ "$headref" = "${symrefcap:-refs/heads/master}" ] && [ $matchcnt -gt 1 ]; then
 569                                                 break
 570                                         fi
 571                                 done <<-EOT
 572                                 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 573                                         LC_ALL=C awk '{print $2}')
 574                                 EOT
 575                                 # Warn if there was more than one match and $symrefcap is empty
 576                                 # or $symrefcap is not the same as $headref since our choice might
 577                                 # differ from the source repository's HEAD
 578                                 if [ $matchcnt -ge 1 ] && [ "$symrefcap" != "$headref" ] &&
 579                                    { [ -n "$symrefcap" ] || [ $matchcnt -gt 1 ]; }; then
 580                                         showheadwarn=1
 581                                 fi
 582                         fi
 583                         if [ -z "$headref" ]; then
 584                                 # If we still don't have a HEAD ref then prefer refs/heads/master
 585                                 # if it exists otherwise take the first refs/heads/...
 586                                 # We do not support having a detached HEAD.
 587                                 # We always warn now because we will be setting HEAD differently
 588                                 # than the source repository had HEAD set
 589                                 showheadwarn=1
 590                                 while read ref; do
 591                                         [ -n "$ref" ] || continue
 592                                         if [ -z "$headref" ] || [ "$ref" = "refs/heads/master" ]; then
 593                                                 headref="$ref"
 594                                         fi
 595                                         [ "$headref" != "refs/heads/master" ] || break
 596                                 done <<-EOT
 597                                 $(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
 598                                         LC_ALL=C awk '{print $2}')
 599                                 EOT
 600                         fi
 601                         # If we STILL do not have a HEAD ref (perhaps the source repository
 602                         # contains only tags) then use refs/heads/master.  It will be invalid
 603                         # but is no worse than we used to do by default and we'll warn about
 604                         # it.  We do not support a HEAD symref to anything other than refs/heads/...
 605                         [ -n "$headref" ] || headref="refs/heads/master"
 606                         git symbolic-ref HEAD "$headref"
 607                         pruneopt=--prune
 608                         [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
 609                         # remember the starting time so we can easily detect new packs for fast-import mirrors
 610                         # we sleep for 1 second after creating .gfipack to make sure all packs are newer
 611                         if is_gfi_mirror_url "$url" && ! [ -e .gfipack ]; then
 612                                 rm -f .gfipack
 613                                 >.gfipack
 614                                 sleep 1
 615                         fi
 616                         GIT_SSL_NO_VERIFY=1 git_ulimit remote update $pruneopt
 617                         if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
 618                                 find -L objects/pack -type f -newer .gfipack -name "pack-$octet20*.pack" -print >>gfi-packs
 619                                 rm -f .gfipack
 620                         fi
 621                 else
 622                         warnempty=1
 623                         git symbolic-ref HEAD "refs/heads/master"
 624                 fi
 625                 rm -f .refs-temp .pkts-temp
 626                 ;;
 627 esac
 628
 629 # Check the max_clone_objects setting now (if set)
 630 if [ "${cfg_max_clone_objects:-0}" != "0" ]; then
 631         objcount="$(git count-objects -v | LC_ALL=C awk 'BEGIN{v=0}/^count:/||/^in-pack:/{v+=$2}END{print v}')" || :
 632         if [ -n "$objcount" ] && [ "$objcount" -gt "$cfg_max_clone_objects" ]; then
 633                 exit_objs="$objcount"
 634                 exit 1 # fail the clone
 635         fi
 636 fi
 637
 638 # The objects subdirectories permissions must be updated now.
 639 # In the case of a dumb http clone, the permissions will not be correct
 640 # (missing group write) despite the core.sharedrepository=1 setting!
 641 # The objects themselves seem to have the correct permissions.
 642 # This problem appears to have been fixed in the most recent git versions.
 643 perms=g+w
 644 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 645 chmod $perms $(find -L objects -maxdepth 1 -type d) 2>/dev/null || :
 646
 647 # We may have just cloned a lot of refs and they will all be
 648 # individual files at this point.  Let's pack them now so we
 649 # can have better performance right from the start.
 650 git pack-refs --all
 651
 652 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
 653 git config gitweb.lastreceive "$(date '+%a, %d %b %Y %T %z')"
 654 git config gitweb.lastchange "$(date '+%a, %d %b %Y %T %z')"
 655 git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 656         --count=1 refs/heads >info/lastactivity || :
 657 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 658
 659 # Don't leave a multi-megabyte useless FETCH_HEAD behind
 660 rm -f FETCH_HEAD
 661
 662 # Last ditch attempt to get a valid HEAD for a non-git source
 663 check_and_set_head || :
 664
 665 # The rest
 666 echo "Final touches..."
 667 git update-server-info
 668 trap "" EXIT
 669
 670 # run gc now unless the clone is empty
 671 if [ -z "$warnempty" ]; then
 672         git config --unset gitweb.lastgc 2>/dev/null || :
 673         rm -f .delaygc .allowgc
 674 fi
 675
 676 emptynote=
 677 [ -z "$warnempty" ] ||
 678 emptynote="
 679 WARNING:  You have mirrored an empty repository.
 680 "
 681 headnote=
 682 [ -z "$showheadwarn" ] || [ -z "$headref" ] ||
 683 headnote="
 684 NOTE:  HEAD has been set to a symbolic ref to \"$headref\".
 685        Use the \"Project settings\" link to choose a different HEAD symref.
 686 "
 687 sizenote=
 688 ! is_gfi_mirror ||
 689 sizenote="
 690 NOTE:  Since this is a mirror of a non-Git source, the initial repository
 691        size may be somewhat larger than necessary.  This will be corrected
 692        shortly.  If you intend to clone this repository you may want to
 693        wait up to 1 hour before doing so in order to receive the more
 694        compact final size.
 695 "
 696 [ -z "$mailaddrs" ] ||
 697 mailref "clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
 698 Congratulations! The clone of project $proj just completed.
 699
 700         * Source URL: $url
 701         * GitWeb interface: $cfg_gitweburl/$proj.git
 702         * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
 703 $emptynote$headnote$sizenote
 704 Have a lot of fun.
 705 EOT
 706
 707 echo "Mirroring finished successfuly!"
 708 # In case this is a re-mirror, lastgc could have been set already so clear it now
 709 git config --unset gitweb.lastgc || :
 710 rm .clone_in_progress
 711 echo "$sizenote@OVER@"