3 # Invoked from taskd/taskd.pl
10 [ "$cfg_permission_control" != "Hooks" ] ||
umask 000
13 # darcs fast-export | git fast-import with error handling
19 { read -r _err1 ||
:; read -r _err2 ||
:; } <<-EOT
21 exec 4>&3 3>&1 1>&4 4>&-
24 "$cfg_basedir"/bin/darcs-fast-export \
25 --export-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
31 --export-marks="$(pwd)/gfi-marks" \
32 --export-pack-edges="$(pwd)/gfi-packs" \
33 --force 3>&- || _e2=$?
39 [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
43 # bzr fast-export | git fast-import with error handling
51 { read -r _err1 ||
:; read -r _err2 ||
:; } <<-EOT
53 exec 4>&3 3>&1 1>&4 4>&-
56 bzr fast-export --plain \
57 --export-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
63 --export-marks="$(pwd)/gfi-marks" \
64 --export-pack-edges="$(pwd)/gfi-packs" \
65 --force 3>&- || _e2=$?
71 [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
77 # We must now close the .clonelog file that is open on stdout and stderr
79 ! [ -d htmlcache
] ||
{ >htmlcache
/changed
; } 2>/dev
/null ||
:
80 failaddrs
="$(config_get owner)" ||
:
81 [ -z "$cfg_admincc" ] ||
[ "$cfg_admincc" = "0" ] ||
[ -z "$cfg_admin" ] ||
82 if [ -z "$failaddrs" ]; then failaddrs
="$cfg_admin"; else failaddrs
="$failaddrs,$cfg_admin"; fi
83 [ -z "$failaddrs" ] ||
86 Condolences. The clone of project $proj just failed.
89 * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
91 The project settings link may be used to adjust the settings
92 and restart the clone in order to try the clone again.
94 if [ -f .clonelog
] && [ -r .clonelog
]; then
98 loglines
=$
(LC_ALL
=C
wc -l <.clonelog
)
99 if [ $loglines -le 203 ]; then
102 head -n 100 .clonelog
104 echo "[ ... elided $(( $loglines - 200 )) middle lines ... ]"
106 tail -n 100 .clonelog
109 } | mailref
"clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone failed" "$failaddrs" ||
:
112 # removes all leftovers from a previous failed clone attempt
113 cleanup_failed_clone
() {
115 # Remove any left-over svn-remote.svn or remote.origin config
116 git config
--remove-section svn-remote.svn
2>/dev
/null ||
:
117 git config
--remove-section remote.origin
2>/dev
/null ||
:
119 # If there is a remote-template.origin section, pre-seed the
120 # remote.origin section with its contents
121 git config
--get-regexp '^remote-template\.origin\..' |
122 while read name value
; do
123 if [ -n "$name" ] && [ -n "$value" ]; then
124 git config
"remote${name#remote-template}" "$value"
128 # Any pre-existing FETCH_HEAD from a previous clone failed or not is
129 # now garbage to be removed
132 # Remove any left-over svn dir from a previous failed attempt
135 # Remove any left-over .darcs dirs from a previous failed attempt
138 # Remove any left-over repo.hg dir from a previous failed attempt
141 # Remove any left-over import/export/temp files from a previous failed attempt
142 rm -f bfe-marks dfe-marks hg2git-heads hg2git-mapping hg2git-marks
* hg2git-state \
143 gfi-marks gfi-packs .pkts-temp .refs-temp
145 # We want a gc right after the clone, so re-enable that just in case.
146 # There's a potential race where we could add it and gc.sh could remove
147 # it, but we'll reunset lastgc just before we remove .delaygc at the end.
148 [ -e .delaygc
] ||
>.delaygc
149 git config
--unset gitweb.lastgc
2>/dev
/null ||
:
151 # Remove all pre-existing refs
153 git for-each-ref
--format='delete %(refname)' | git_updateref_stdin
2>/dev
/null ||
:
155 # The initial state before a clone starts has HEAD as a symbolic-ref to master
156 git symbolic-ref HEAD refs
/heads
/master
158 # HEAD is no longer "ok"
159 git config
--unset girocco.headok
2>/dev
/null ||
:
161 # We, perhaps, ought to remove any packs/loose objects now, but the next gc
162 # will get rid of any extras. Also, if we're recloning the same thing, any
163 # preexisting packs/loose objects containing what we're recloning will only
164 # speed up the reclone by avoiding some disk writes. So we don't kill them.
166 # It's just remotely possible that a bunch of failures in a row could
167 # create a big mess that just keeps growing and growing...
168 # Trigger a .needsgc if that happens.
169 check_and_set_needsgc
173 cd "$cfg_reporoot/$proj.git"
176 ! [ -e .delaygc
] ||
>.allowgc ||
:
178 trap "echo '@OVER@'; touch .clone_failed; send_clone_failed" EXIT
179 echo "Project: $proj"
180 echo " Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
182 [ -n "$cfg_mirror" ] ||
{ echo "Mirroring is disabled" >&2; exit 1; }
183 url
="$(config_get baseurl)" ||
:
184 case "$url" in *" "*|
*" "*|
"")
185 echo "Bad mirror URL (\"$url\")"
191 # Record original mirror type for use by update.sh
192 mirror_type
="$(get_url_mirror_type "$url")"
193 git config girocco.mirrortype
"$mirror_type"
195 echo "Mirroring from URL \"$url\""
198 if [ "$cfg_project_owners" = "source" ]; then
199 config
set owner
"$(ls -ldH "${url#file://}" 2>/dev/null | LC_ALL=C awk '{print $3}')"
202 mailaddrs
="$(config_get owner)" ||
:
203 [ -z "$cfg_admin" ] ||
204 if [ -z "$mailaddrs" ]; then mailaddrs
="$cfg_admin"; else mailaddrs
="$mailaddrs,$cfg_admin"; fi
206 # Make sure we don't get any unwanted loose objects
207 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
208 # tweak its configuration to prevent that
209 git_add_config
'fetch.unpackLimit=1'
210 # Note the git config documentation is wrong
211 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
212 git_add_config
'transfer.unpackLimit=1'
213 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
214 git_add_config
'fastimport.unpackLimit=0'
217 echo "Initiating mirroring..."
222 # remember the starting time so we can easily combine fetched loose objects
223 # we sleep for 1 second after creating .needspack to make sure all objects are newer
224 if ! [ -e .needspack
]; then
231 svn
://* | svn
+http
://* | svn
+https
://* | svn
+file://* | svn
+ssh://*)
232 [ -n "$cfg_mirror_svn" ] ||
{ echo "Mirroring svn is disabled" >&2; exit 1; }
233 # Allow the username to be specified in the "svn-credential.svn.username"
234 # property and the password in the "svn-credential.svn.password" property
235 # Use an 'anonsvn' username by default as is commonly used for anonymous svn
236 # Default the password to the same as the username
237 # The password property will be ignored unless a username has been specified
238 if svnuser
="$(git config --get svn-credential.svn.username)" && [ -n "$svnuser" ]; then
239 if ! svnpass
="$(git config --get svn-credential.svn.password)"; then
244 case "$url1" in ?
*"@"?
*)
245 urlsch
="${url%%://*}"
246 url
="$urlsch://${url#*@}"
249 # As a fallback, check in the URL, just in case
253 case "$url1" in ?
*"@"?
*)
254 urlsch
="${url%%://*}"
255 url
="$urlsch://${url#*@}"
257 svnuser
="${url1%%:*}"
258 if [ -n "$svnuser" ]; then
260 case "$url1" in *":"*)
265 if [ -z "$svnuser" ]; then
270 GIT_ASKPASS_PASSWORD
="$svnpass"
271 export GIT_ASKPASS_PASSWORD
272 # We just remove svn+ here, so svn+http://... becomes http://...
273 # We also remove a trailing '/' to match what git-svn will do
274 case "$url" in svn
+ssh://*) svnurl
="$url";; *) svnurl
="${url#svn+}";; esac
276 # We require svn info to succeed on the URL otherwise it's
277 # simply not a valid URL and without using -s on the init it
278 # will not otherwise be tested until the fetch
279 svn
--non-interactive --username "$svnuser" --password "$svnpass" info
"$svnurl" >/dev
/null
280 # We initially use -s for the init which will possibly shorten
281 # the URL. However, the shortening can fail if a password is
282 # not required for the longer version but is for the shorter,
283 # so try again without -s if the -s version fails.
284 # We must use GIT_DIR=. here or ever so "helpful" git-svn will
285 # create a .git subdirectory!
286 GIT_DIR
=. git svn init
--username="$svnuser" --prefix "" -s "$svnurl" </dev
/null ||
287 GIT_DIR
=. git svn init
--username="$svnuser" --prefix "" "$svnurl" </dev
/null
288 # We need to remember this url so we can detect changes because
289 # ever so "helpful" git-svn may shorten it!
290 config_set svnurl
"$svnurl"
291 # At this point, since we asked for a standard layout (-s) git-svn
292 # may have been "helpful" and adjusted our $svnurl to a prefix and
293 # then glued the removed suffix onto the front of any svn-remote.svn.*
294 # config items. We could avoid this by not using the '-s' option
295 # but then we might not get all the history. If, for example, we
296 # are cloning an http://svn.example.com/repos/public repository that
297 # early in its history moved trunk => public/trunk we would miss that
298 # earlier history without allowing the funky shorten+prefix behavior.
299 # So we read back the svn-remote.svn.fetch configuration and compute
300 # the prefix. This way we are sure to get the correct prefix.
301 gitsvnurl
="$(git config --get svn-remote.svn.url)" ||
:
302 gitsvnfetch
="$(git config --get-all svn-remote.svn.fetch | tail -1)" ||
:
303 gitsvnprefix
="${gitsvnfetch%%:*}"
304 gitsvnsuffix
="${gitsvnprefix##*/}"
305 gitsvnprefix
="${gitsvnprefix%$gitsvnsuffix}"
306 # Ask git-svn to store everything in the normal non-remote
307 # locations being careful to use the correct prefix
308 git config
--replace-all svn-remote.svn.fetch
"${gitsvnprefix}trunk:refs/heads/master"
309 git config
--replace-all svn-remote.svn.branches
"${gitsvnprefix}branches/*:refs/heads/*"
310 git config
--replace-all svn-remote.svn.tags
"${gitsvnprefix}tags/*:refs/tags/*"
311 # look for additional non-standard directories to fetch
312 # check for standard layout at the same time
315 svn
--non-interactive --username "$svnuser" --password "$svnpass" ls "$gitsvnurl/${gitsvnprefix}" 2>/dev
/null |
316 { while read file; do case $file in
317 # skip the already-handled standard ones and any with a space or tab
319 trunk
/|branches
/|tags
/) foundstd
=1;;
320 # only fetch extra directories from the $svnurl root (not any files)
321 *?
/) git config
--add svn-remote.svn.fetch \
322 "${gitsvnprefix}${file%/}:refs/heads/${file%/}";;
325 # if files found and no standard directories present use a simpler layout
326 if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then
327 git config
--unset svn-remote.svn.branches
328 git config
--unset svn-remote.svn.tags
329 git config
--replace-all svn-remote.svn.fetch
':refs/heads/master'
332 # Again, be careful to use GIT_DIR=. here or else new .git subdirectory!
333 GIT_DIR
=. git svn fetch
--log-window-size=$var_log_window_size --username="$svnuser" --quiet </dev
/null
334 # git svn does not preserve group permissions in the svn subdirectory
335 chmod -R ug
+rw
,o
+r svn
336 # git svn also leaves behind ref turds that end with @nnn
337 # We get rid of them now
338 git for-each-ref
--format='%(refname)' |
339 LC_ALL
=C
sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
341 unset GIT_ASKPASS_PASSWORD
343 darcs
://* | darcs
+http
://* | darcs
+https
://*)
344 [ -n "$cfg_mirror_darcs" ] ||
{ echo "Mirroring darcs is disabled" >&2; exit 1; }
346 darcs
://*) darcsurl
="http://${url#darcs://}";;
347 *) darcsurl
="${url#darcs+}";;
349 git_darcs_fetch
"$darcsurl"
352 [ -n "$cfg_mirror_bzr" ] ||
{ echo "Mirroring bzr is disabled" >&2; exit 1; }
353 # we just remove bzr:// here, a typical bzr url is just
355 bzrurl
="${url#bzr://}"
356 git_bzr_fetch
"$bzrurl"
358 hg
+http
://* | hg
+https
://* | hg
+file://* | hg
+ssh://*)
359 [ -n "$cfg_mirror_hg" ] ||
{ echo "Mirroring hg is disabled" >&2; exit 1; }
360 # We just remove hg+ here, so hg+http://... becomes http://...
362 # Perform the initial hg clone
363 hg clone
-U "$hgurl" "$(pwd)/repo.hg"
364 # Do the fast-export | fast-import
368 # We manually add remote.origin.url and remote.origin.fetch
369 # to simulate a `git remote add --mirror=fetch` since that's
370 # not available until Git 1.7.5 and this way we guarantee we
371 # always get exactly the intended configuration and nothing else.
372 git config remote.origin.url
"$url"
373 if ! is_gfi_mirror_url
"$url" && [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" = "true" ]; then
374 git config
--replace-all remote.origin.fetch
"+refs/heads/*:refs/heads/*"
375 git config
--add remote.origin.fetch
"+refs/tags/*:refs/tags/*"
376 git config
--add remote.origin.fetch
"+refs/notes/*:refs/notes/*"
377 git config
--add remote.origin.fetch
"+refs/top-bases/*:refs/top-bases/*"
378 git config
--bool girocco.lastupdateclean true
380 git config
--replace-all remote.origin.fetch
"+refs/*:refs/*"
381 git config
--bool girocco.lastupdateclean false
383 # Set the correct HEAD symref by using ls-remote first
384 GIT_SSL_NO_VERIFY
=1 GIT_TRACE_PACKET
=1 git ls-remote origin
>.refs-temp
2>.pkts-temp ||
386 # Since everything was redirected, on failure there'd be no output,
387 # so let's make some failure output
390 echo "git ls-remote \"$url\" failed"
393 # Compensate for git() {} side effects
394 unset GIT_TRACE_PACKET
395 # If the server is running at least Git 1.8.4.3 then it will send us the actual
396 # symref for HEAD. If we are running at least Git 1.7.5 then we can snarf that
397 # out of the packet trace data.
398 if [ -s .refs-temp
]; then
399 # Nothing to do unless the remote repository has at least 1 ref
400 # See if we got a HEAD ref
401 head="$(LC_ALL=C grep -E "^
$octet20$hexdig*[ $tab]+HEAD\$
" <.refs-temp | LC_ALL=C awk '{print $1}')"
402 # If the remote has HEAD set to a symbolic ref that does not exist
403 # then we will not receive a HEAD ref in the ls-remote output
407 if [ -n "$head" ]; then
408 symrefcap
="$(LC_ALL=C sed -ne <.pkts-temp \
409 "/packet
:.
*git
<.
*[ $tab]symref
="'HEAD:refs\/heads\/'"[^
$tab]/\
410 {s
/^.
*[ $tab]symref
="'HEAD:\(refs\/heads\/'"[^
$tab][^
$tab]*"'\).*$/\1/;p;}')"
411 # prefer $symrefcap (refs/heads/master if no $symrefcap) if it
412 # matches HEAD otherwise take the first refs/heads/... match
415 [ -n "$ref" ] ||
continue
416 matchcnt
=$
(( $matchcnt + 1 ))
417 if [ -z "$headref" ] ||
[ "$ref" = "${symrefcap:-refs/heads/master}" ]; then
420 if [ "$headref" = "${symrefcap:-refs/heads/master}" ] && [ $matchcnt -gt 1 ]; then
424 $(LC_ALL=C grep -E "^$head[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
425 LC_ALL=C awk '{print $2}')
427 # Warn if there was more than one match and $symrefcap is empty
428 # or $symrefcap is not the same as $headref since our choice might
429 # differ from the source repository's HEAD
430 if [ $matchcnt -ge 1 ] && [ "$symrefcap" != "$headref" ] &&
431 { [ -n "$symrefcap" ] ||
[ $matchcnt -gt 1 ]; }; then
435 if [ -z "$headref" ]; then
436 # If we still don't have a HEAD ref then prefer refs/heads/master
437 # if it exists otherwise take the first refs/heads/...
438 # We do not support having a detached HEAD.
439 # We always warn now because we will be setting HEAD differently
440 # than the source repository had HEAD set
443 [ -n "$ref" ] ||
continue
444 if [ -z "$headref" ] ||
[ "$ref" = "refs/heads/master" ]; then
447 [ "$headref" != "refs/heads/master" ] ||
break
449 $(LC_ALL=C grep -E "^$octet20$hexdig*[ $tab]+refs/heads/[^ $tab]+\$" <.refs-temp |
450 LC_ALL=C awk '{print $2}')
453 # If we STILL do not have a HEAD ref (perhaps the source repository
454 # contains only tags) then use refs/heads/master. It will be invalid
455 # but is no worse than we used to do by default and we'll warn about
456 # it. We do not support a HEAD symref to anything other than refs/heads/...
457 [ -n "$headref" ] || headref
="refs/heads/master"
458 git symbolic-ref HEAD
"$headref"
460 [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt
=
461 # remember the starting time so we can easily detect new packs for fast-import mirrors
462 # we sleep for 1 second after creating .gfipack to make sure all packs are newer
463 if is_gfi_mirror_url
"$url" && ! [ -e .gfipack
]; then
468 GIT_SSL_NO_VERIFY
=1 git remote update
$pruneopt
469 if [ -e .gfipack
] && is_gfi_mirror_url
"$url"; then
470 find -L objects
/pack
-type f
-newer .gfipack
-name "pack-$octet20*.pack" -print >>gfi-packs
475 git symbolic-ref HEAD
"refs/heads/master"
477 rm -f .refs-temp .pkts-temp
481 # The objects subdirectories permissions must be updated now.
482 # In the case of a dumb http clone, the permissions will not be correct
483 # (missing group write) despite the core.sharedrepository=1 setting!
484 # The objects themselves seem to have the correct permissions.
485 # This problem appears to have been fixed in the most recent git versions.
487 [ "$cfg_permission_control" != "Hooks" ] || perms
=go
+w
488 chmod $perms $
(find -L objects
-maxdepth 1 -type d
) 2>/dev
/null ||
:
490 # We may have just cloned a lot of refs and they will all be
491 # individual files at this point. Let's pack them now so we
492 # can have better performance right from the start.
495 # Initialize gitweb.lastreceive, gitweb.lastchange and info/lastactivity
496 git config gitweb.lastreceive
"$(date '+%a, %d %b %Y %T %z')"
497 git config gitweb.lastchange
"$(date '+%a, %d %b %Y %T %z')"
498 git for-each-ref
--sort=-committerdate --format='%(committerdate:iso8601)' \
499 --count=1 refs
/heads
>info
/lastactivity ||
:
500 ! [ -d htmlcache
] ||
{ >htmlcache
/changed
; } 2>/dev
/null ||
:
502 # Don't leave a multi-megabyte useless FETCH_HEAD behind
505 # Last ditch attempt to get a valid HEAD for a non-git source
506 check_and_set_head ||
:
509 echo "Final touches..."
510 git update-server-info
513 # run gc now unless the clone is empty
514 if [ -z "$warnempty" ]; then
515 git config
--unset gitweb.lastgc
2>/dev
/null ||
:
516 rm -f .delaygc .allowgc
520 [ -z "$warnempty" ] ||
522 WARNING: You have mirrored an empty repository.
525 [ -z "$showheadwarn" ] ||
[ -z "$headref" ] ||
527 NOTE: HEAD has been set to a symbolic ref to \"$headref\".
528 Use the \"Project settings\" link to choose a different HEAD symref.
533 NOTE: Since this is a mirror of a non-Git source, the initial repository
534 size may be somewhat larger than necessary. This will be corrected
535 shortly. If you intend to clone this repository you may want to
536 wait up to 1 hour before doing so in order to receive the more
539 [ -z "$mailaddrs" ] ||
540 mailref
"clone@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj clone completed" "$mailaddrs" <<EOT || :
541 Congratulations! The clone of project $proj just completed.
544 * GitWeb interface: $cfg_gitweburl/$proj.git
545 * Project settings: $cfg_webadmurl/editproj.cgi?name=$(echo "$proj" | LC_ALL=C sed -e 's/[+]/%2B/g')
546 $emptynote$headnote$sizenote
550 echo "Mirroring finished successfuly!"
551 # In case this is a re-mirror, lastgc could have been set already so clear it now
552 git config
--unset gitweb.lastgc ||
:
553 rm .clone_in_progress
554 echo "$sizenote@OVER@"