jobd/update.sh

   1 #!/bin/sh
   2
   3 . @basedir@/shlib.sh
   4 . @basedir@/jobd/updategc-util-functions.sh
   5
   6 set -e
   7
   8 if [ $# -ne 1 ]; then
   9         echo "Usage: update.sh projname" >&2
  10         exit 1
  11 fi
  12
  13 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
  14 datefmt='+%a, %d %b %Y %T %z'
  15
  16 git_fetch_q_progress() (
  17         _setexport_gitvars
  18         if [ "${cfg_max_file_size512:-0}" != "0" ]; then
  19                 GIT_BIN="'$cfg_basedir/bin/ulimit512' -i -f '$cfg_max_file_size512' -- '$cfg_git_bin'" &&
  20                 export GIT_BIN
  21         fi
  22         PATH="$var_git_exec_path:$cfg_basedir/bin:$PATH"
  23         export PATH
  24         exec @basedir@/jobd/git-fetch-q-progress.sh "$@"
  25 )
  26
  27 # freshen_loose_objects full-sha ...
  28 # if "$n" is a loose object, set its modification time to now
  29 # otherwise silently do nothing with no error.  To facilitate conversion
  30 # of mirror projects to push projects we also add group write  permission.
  31 freshen_loose_objects() {
  32         _list=
  33         for _sha; do
  34                 _fn="${_sha#??}"
  35                 _shard="${_sha%$_fn}"
  36                 _list="$_list objects/$_shard/$_fn"
  37         done
  38         if [ -n "$_list" ]; then
  39                 chmod ug+w $_list 2>/dev/null || :
  40                 touch -c $_list 2>/dev/null || :
  41         fi
  42 }
  43
  44 # darcs fast-export | git fast-import with error handling
  45 git_darcs_fetch() (
  46         set_utf8_locale
  47         _err1=
  48         _err2=
  49         exec 3>&1
  50         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  51         $(
  52                 exec 4>&3 3>&1 1>&4 4>&-
  53                 {
  54                         _e1=0
  55                         "$cfg_basedir"/bin/darcs-fast-export \
  56                                 --export-marks="$(pwd)/dfe-marks" \
  57                                 --import-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  58                         echo $_e1 >&3
  59                 } |
  60                 {
  61                         _e2=0
  62                         git_ulimit fast-import \
  63                                 --export-marks="$(pwd)/gfi-marks" \
  64                                 --export-pack-edges="$(pwd)/gfi-packs" \
  65                                 --import-marks="$(pwd)/gfi-marks" \
  66                                 --force 3>&- || _e2=$?
  67                         echo $_e2 >&3
  68                 }
  69         )
  70         EOT
  71         exec 3>&-
  72         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  73         return $?
  74 )
  75
  76 # bzr fast-export | git fast-import with error handling
  77 git_bzr_fetch() (
  78         set_utf8_locale
  79         BZR_LOG=/dev/null
  80         export BZR_LOG
  81         _err1=
  82         _err2=
  83         exec 3>&1
  84         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  85         $(
  86                 exec 4>&3 3>&1 1>&4 4>&-
  87                 {
  88                         _e1=0
  89                         bzr fast-export --plain \
  90                                 --export-marks="$(pwd)/bfe-marks" \
  91                                 --import-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  92                         echo $_e1 >&3
  93                 } |
  94                 {
  95                         _e2=0
  96                         git_ulimit fast-import \
  97                                 --export-marks="$(pwd)/gfi-marks" \
  98                                 --export-pack-edges="$(pwd)/gfi-packs" \
  99                                 --import-marks="$(pwd)/gfi-marks" \
 100                                 --force 3>&- || _e2=$?
 101                         echo $_e2 >&3
 102                 }
 103         )
 104         EOT
 105         exec 3>&-
 106         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
 107         return $?
 108 )
 109
 110 # On return a "$lockf" will have been created that must be removed when gc is done
 111 lock_update() {
 112         v_lock_file _lockresult "update.pid" || {
 113                 echo >&2 "[$proj] $_lockresult"
 114                 exit 1
 115         }
 116         lockf="$_lockresult"
 117 }
 118
 119 # output all loose object ids, one per line, to stdout INCLUDING "/" shard separator
 120 # look in "$1" (default "objects")
 121 list_loose_sharded_objects() (
 122         cd "${1:-objects}" || return 1
 123         objdirs="$(echo $octet)"
 124         [ "$objdirs" != "$octet" ] || return 0
 125         find -L $objdirs -mindepth 1 -maxdepth 1 -type f -name "$octet19*" -print
 126 )
 127
 128 # Migrate any and all objects/packs/ref-changes from $incoming_fetch/$incoming_objs
 129 # Does NOT do anything with reflogs, those should already be taken care of elsewhere
 130 unquarantine_updates() {
 131         # just make sure everthing is copacetic first
 132         [ -n "$incoming_fetch" ] && [ -d "$incoming_fetch" ] &&
 133         [ -n "$incoming_objs" ] && [ -d "$incoming_objs" ] || {
 134                 echo >&2 "[$proj] unquarantine failed"
 135                 exit 1
 136         }
 137         _ifd="$(git --git-dir="$incoming_fetch" rev-parse --git-dir 2>/dev/null)" || :
 138         [ -z "$_ifd" ] || _ifd="$(cd "$_ifd" && pwd -P)"
 139         [ -n "$_ifd" ] && [ "$_ifd" = "$incoming_fetch" ] || {
 140                 echo >&2 "[$proj] unquarantine failed"
 141                 exit 1
 142         }
 143
 144         # both $incoming_fetch and . must have all their refs packed
 145         git --git-dir="$incoming_fetch" pack-refs --all --prune
 146         git --git-dir=. pack-refs --all --prune
 147
 148         # now every loose object and pack must be migrated out of quarantine
 149         _objd="$(cd "$PWD/objects" && pwd -P)"
 150         # packs first
 151         if [ -d "$incoming_objs/pack" ]; then
 152                 if [ ! -d "$_objd/pack" ]; then
 153                         mkdir -p "$_objd/pack"
 154                         chmod "$dperms" "$_objd/pack" >/dev/null 2>&1 || :
 155                 fi
 156                 while read -r _pckf && [ -n "${_pckf%.pack}" ]; do
 157                         _pckf="${_pckf%.pack}"
 158                         rename_pack "$_pckf" "$_objd/pack/${_pckf##*/}"
 159                         chmod "$fperms" "$_objd/pack/${_pckf##*/}".?* >/dev/null 2>&1 || :
 160                 done <<LIST_PACKS
 161 $(list_packs --exclude-no-idx "$incoming_objs/pack")
 162 LIST_PACKS
 163         fi
 164         # now loose objects
 165         # (use a subshell for loose objects as there could potentially be many
 166         # whereas there will normally be just one pack)
 167         list_loose_sharded_objects "$incoming_objs" |
 168         while read -r _objf && [ -n "$_objf" ] && [ "${#_objf}" -ge 41 ]; do
 169                 if [ ! -d "$_objd/${_objf%%/*}" ]; then
 170                         mkdir -p "$_objd/${_objf%%/*}"
 171                         chmod "$dperms" "$_objd/${_objf%%/*}" >/dev/null 2>&1 || :
 172                 fi
 173                 ln "$incoming_objs/$_objf" "$_objd/$_objf" >/dev/null 2>&1 ||
 174                 dupe_file "$incoming_objs/$_objf" "$_objd/$_objf" "tmp_obj_" >/dev/null 2>&1 ||
 175                 [ -f "$_objd/$_objf" ] || {
 176                         echo >&2 "[$proj] unable to unquarantine object $_objf"
 177                         exit 1
 178                 }
 179                 chmod "$fperms" "$_objd/$_objf" >/dev/null 2>&1 || :
 180                 rm -f "$incoming_objs/$_objf"
 181         done || exit 1
 182
 183         # now the refs
 184         # simply replace the packed-refs file
 185         # but do it atomically and make sure it's on the same file system first
 186         rm -f "$PWD/packed-refs.$$"
 187         cat "$incoming_fetch/packed-refs" >"$PWD/packed-refs.$$"
 188         mv -f "$PWD/packed-refs.$$" "$PWD/packed-refs"
 189         rm -f "$PWD/packed-refs.$$"
 190 }
 191
 192 # Create a "quarantine" area to fetch into
 193 # This is set up similarly to the way the "repack" directory is set
 194 # up for gc in that it's a subdirectory that's a whole "git" directory
 195 # but it uses the existing objects directory as an alternate and its
 196 # own objects subdirectory is a symlink to a subdirectory of the real
 197 # objects directory (to guarantee that packs/objects can be moved rather
 198 # than copied).  It starts out with a copy of all of the project's refs.
 199 # A successful fetch will "unquarantine" fetched objects/packs + ref changes
 200 create_quarantine() {
 201         incoming_objs="$(mktemp -d "$PWD/objects/incoming-XXXXXX")"
 202         incoming_objs="$(cd "$incoming_objs" && pwd -P)"
 203         chmod "$dperms" "$incoming_objs"
 204         mkdir "$incoming_objs/pack"
 205         mkdir "$incoming_objs/info"
 206         printf '%s\n' "$PWD/objects" >"$incoming_objs/info/alternates"
 207         incoming_fetch="$(mktemp -d "$PWD/incoming-XXXXXX")"
 208         incoming_fetch="$(cd "$incoming_fetch" && pwd -P)"
 209         chmod "$dperms" "$incoming_fetch"
 210         ln -s "$incoming_objs" "$incoming_fetch/objects"
 211         mkdir "$incoming_fetch/refs"
 212         ln -s "$PWD/config" "$incoming_fetch/config"
 213         git for-each-ref --format='%(objectname) %(refname)' >"$incoming_fetch/packed-refs"
 214         cat HEAD >"$incoming_fetch/HEAD"
 215         # Make sure the incoming packed-refs file is properly peeled
 216         git --git-dir="$incoming_fetch" pack-refs --all --prune
 217         # link to svn if it exists
 218         [ ! -d svn ] || ln -s "$PWD/svn" "$incoming_fetch/svn"
 219         use_quarantine=1
 220 }
 221
 222 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 0; }
 223
 224 if [ "$cfg_permission_control" != "Hooks" ]; then
 225         umask 002
 226         fperms=0664
 227         dperms=2775
 228 else
 229         umask 000
 230         fperms=0666
 231         dperms=2777
 232 fi
 233 clean_git_env
 234
 235 proj="${1%.git}"
 236 cd "$cfg_reporoot/$proj.git"
 237
 238 # Activate a mini-gc if needed
 239 # We do this here as well as after a successful fetch so that if we're stuck
 240 # in a fetch loop where fetches are succeeding in fetching new packs but the
 241 # ref update is failing for some reason (perhaps a non-commit under refs/heads)
 242 # and a previous invokation therefore had a "bang" exit then we will still
 243 # get the .needsgc flag set in a timely fashion to avoid excess pack build up.
 244 check_and_set_needsgc
 245
 246 use_quarantine=
 247 bang_log=
 248 incoming_fetch=
 249 incoming_objs=
 250 lockf=
 251 cleanup_exit() {
 252         ec=$?
 253         if [ $ec != 0 ]; then
 254                 echo "update failed dir: $PWD" >&2
 255         fi
 256         [ -z "$incoming_fetch" ] || rm -rf "$incoming_fetch"
 257         [ -z "$incoming_objs" ] || rm -rf "$incoming_objs"
 258         [ -z "$bang_log" ] || rm -f "$bang_log"
 259         [ -z "$lockf" ] || rm -f "$lockf"
 260 }
 261 bang_exit() { cleanup_exit; }
 262 trap 'cleanup_exit' EXIT
 263 trap 'exit 129' HUP
 264 trap 'exit 130' INT
 265 trap 'exit 131' QUIT
 266 trap 'exit 134' ABRT
 267 trap 'exit 141' PIPE
 268 trap 'exit 142' ALRM
 269 trap 'exit 143' TERM
 270
 271 if [ "${force_update:-0}" = "0" ] && check_interval lastrefresh $cfg_min_mirror_interval; then
 272         progress "= [$proj] update skip (last at $(config_get lastrefresh))"
 273         exit 0
 274 fi
 275 if [ -e .nofetch ]; then
 276         progress "x [$proj] update disabled (.nofetch exists)"
 277         exit 0
 278 fi
 279 lock_update
 280 progress "+ [$proj] update ($(date))"
 281
 282 # Any pre-existing FETCH_HEAD from a previous fetch, failed or not, is garbage
 283 rm -f FETCH_HEAD
 284
 285 # Remove any stale ref locks
 286 clear_stale_ref_locks
 287
 288 # Remove any stale incoming-* object quarantine directories that are
 289 # more than 12 hours old.  These are new with Git >= 2.11.0.
 290 # But we also create our own during the fetch process as Git's quarantine
 291 # only applies to incoming receive-pack which we imitate for our fetch.
 292 find -L . objects -maxdepth 1 -type d -name 'incoming-?*' -mmin +720 \
 293         -exec rm -rf '{}' + || :
 294
 295 # A previous failed update attempt can leave a huge tmp_pack_XXXXXX file behind.
 296 # Since no pushes are allowed to mirrors, we know that any such files that exist
 297 # at this point in time are garbage and can be safely deleted, we do not even
 298 # need to check how old they are.  A tmp_idx_XXXXXX file is also created during
 299 # the later stages of the fetch process, so we kill any of those as well.
 300 find -L objects/pack -maxdepth 1 -type f -name "tmp_pack_?*" -exec rm -f '{}' + || :
 301 find -L objects/pack -maxdepth 1 -type f -name "tmp_idx_?*" -exec rm -f '{}' + || :
 302
 303 # Make sure we have a reflogs subdirectory and abort the update if not
 304 # This should not count as a normal "bang" failure if unsuccessful
 305 [ -d reflogs ] || mkdir -p reflogs >/dev/null 2>&1 || :
 306 [ -d reflogs ]
 307
 308 keep_bang_log=
 309 do_check_after_refs=1
 310 bang_setup
 311 bang_action="update"
 312 bang_trap() {
 313         if [ -n "$1" ]; then
 314                 # Throttle retries
 315                 # Since gitweb shows the .last_refresh date, it's safe to update
 316                 # gitweb.lastrefresh to throttle the updates w/o corrupting the
 317                 # last refresh date display on the gitweb summary page
 318                 # It's therefore important that we do NOT touch .last_refresh here
 319                 config_set lastrefresh "$(date "$datefmt")"
 320         fi
 321 }
 322
 323 bang echo "Project: $proj"
 324 bang echo "   Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
 325 bang echo ""
 326 mail="$(config_get owner)" || :
 327 url="$(config_get baseurl)" || :
 328 case "$url" in *"       "*|*" "*|"")
 329         bang_eval 'echo "Bad mirror URL (\"$url\")"; ! :'
 330         exit 1
 331 esac
 332 bang echo "Mirroring from URL \"$url\""
 333 bang echo ""
 334 statusok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 335 mailaddrs=
 336 [ "$statusok" = "false" ] || [ -z "$mail" ] || mailaddrs="$mail"
 337 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 338 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 339
 340 bang_eval "git for-each-ref --format '%(refname) %(objectname)' >.refs-temp"
 341 bang_eval "LC_ALL=C sort -b -k1,1 <.refs-temp >.refs-before"
 342
 343 check_after_refs() {
 344         [ -n "$do_check_after_refs" ] || return 0
 345         bang_eval "git for-each-ref --format '%(refname) %(objectname)' >.refs-temp"
 346         bang_eval "LC_ALL=C sort -b -k1,1 <.refs-temp >.refs-after"
 347         refschanged=
 348         cmp -s .refs-before .refs-after || refschanged=1
 349         do_check_after_refs=
 350 }
 351
 352 ! [ -e .delaygc ] || >.allowgc || :
 353
 354 # Make sure we don't get any unwanted loose objects
 355 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
 356 # tweak its configuration to prevent that
 357 git_add_config 'fetch.unpackLimit=1'
 358 # Note the git config documentation is wrong
 359 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
 360 git_add_config 'transfer.unpackLimit=1'
 361 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
 362 git_add_config 'fastimport.unpackLimit=0'
 363
 364 # remember the starting time so we can easily combine fetched loose objects
 365 # we sleep for 1 second after creating .needspack to make sure all objects are newer
 366 if ! [ -e .needspack ]; then
 367         rm -f .needspack
 368         >.needspack
 369         sleep 1
 370 fi
 371
 372 case "$url" in
 373         svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 374                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 0; }
 375                 # Allow the username to be specified in the "svn-credential.svn.username"
 376                 # property and the password in the "svn-credential.svn.password" property
 377                 # Use an 'anonsvn' username by default as is commonly used for anonymous svn
 378                 # Default the password to the same as the username
 379                 # The password property will be ignored unless a username has been specified
 380                 if svnuser="$(git config --get svn-credential.svn.username)" && [ -n "$svnuser" ]; then
 381                         if ! svnpass="$(git config --get svn-credential.svn.password)"; then
 382                                 svnpass="$svnuser"
 383                         fi
 384                         url1="${url#*://}"
 385                         url1="${url1%%/*}"
 386                         case "$url1" in ?*"@"?*)
 387                                 urlsch="${url%%://*}"
 388                                 url="$urlsch://${url#*@}"
 389                         esac
 390                 else
 391                         # As a fallback, check in the URL, just in case
 392                         url1="${url#*://}"
 393                         url1="${url1%%/*}"
 394                         svnuser=
 395                         case "$url1" in ?*"@"?*)
 396                                 urlsch="${url%%://*}"
 397                                 url="$urlsch://${url#*@}"
 398                                 url1="${url1%%@*}"
 399                                 svnuser="${url1%%:*}"
 400                                 if [ -n "$svnuser" ]; then
 401                                         svnpass="$svnuser"
 402                                         case "$url1" in *":"*)
 403                                                 svnpass="${url1#*:}"
 404                                         esac
 405                                 fi
 406                         esac
 407                         if [ -z "$svnuser" ]; then
 408                                 svnuser="anonsvn"
 409                                 svnpass="anonsvn"
 410                         fi
 411                 fi
 412                 GIT_ASKPASS_PASSWORD="$svnpass"
 413                 export GIT_ASKPASS_PASSWORD
 414                 # Update the git svn url to match baseurl but be cognizant of any
 415                 # needed prefix changes.  See the comments in taskd/clone.sh about
 416                 # why we need to put up with a prefix in the first place.
 417                 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
 418                 svnurl="${svnurl%/}"
 419                 svnurlold="$(config_get svnurl)" || :
 420                 if [ "$svnurl" != "$svnurlold" ]; then
 421                         # We better already have an svn-remote.svn.fetch setting
 422                         bang test -n "$(git config --get-all svn-remote.svn.fetch)" || :
 423                         # the only way to truly know what the proper prefix is
 424                         # is to attempt a fresh git-svn init -s on the new url
 425                         rm -rf svn-new-url || :
 426                         # We require svn info to succeed on the URL otherwise it's
 427                         # simply not a valid URL and without using -s on the init it
 428                         # will not otherwise be tested until the fetch
 429                         bang eval 'svn --non-interactive --username "$svnuser" --password "$svnpass" info "$svnurl" >/dev/null'
 430                         bang mkdir svn-new-url
 431                         GIT_DIR=svn-new-url bang git init --bare --quiet
 432                         # We initially use -s for the init which will possibly shorten
 433                         # the URL.  However, the shortening can fail if a password is
 434                         # not required for the longer version but is for the shorter,
 435                         # so try again without -s if the -s version fails.
 436                         cmdstr='git svn init --username="$svnuser" --prefix "" -s "$svnurl" <"$mtlinesfile" >/dev/null 2>&1 || '
 437                         cmdstr="$cmdstr"'git svn init --username="$svnuser" --prefix "" "$svnurl" <"$mtlinesfile" >/dev/null 2>&1'
 438                         GIT_DIR=svn-new-url bang eval "$cmdstr"
 439                         gitsvnurl="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.url)" || :
 440                         gitsvnfetch="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.fetch)" || :
 441                         gitsvnprefixnew="${gitsvnfetch%%:*}"
 442                         gitsvnsuffixnew="${gitsvnprefixnew##*/}"
 443                         gitsvnprefixnew="${gitsvnprefixnew%$gitsvnsuffixnew}"
 444                         rm -rf svn-new-url || :
 445                         # Using GIT_DIR= with bang leaves it set to svn-new-url, so reset it to .
 446                         GIT_DIR=.
 447                         if [ "$gitsvnurl" != "$(git config --get svn-remote.svn.url || :)" ]; then
 448                                 # The url has been changed.
 449                                 # We must update the url and replace the prefix on all config items
 450                                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | head -1)" || :
 451                                 gitsvnprefixold="${gitsvnfetch%%:*}"
 452                                 gitsvnsuffixold="${gitsvnprefixold##*/}"
 453                                 gitsvnprefixold="${gitsvnprefixold%$gitsvnsuffixold}"
 454                                 git config --remove-section 'svn-remote.svnnew' 2>/dev/null || :
 455                                 git config 'svn-remote.svnnew.url' "$gitsvnurl"
 456                                 git config --get-regexp '^svn-remote\.svn\.' |
 457                                 while read -r sname sval; do
 458                                         case "$sname" in
 459                                         svn-remote.svn.fetch|svn-remote.svn.branches|svn-remote.svn.tags)
 460                                                 sname="${sname#svn-remote.svn.}"
 461                                                 sval="${sval#$gitsvnprefixold}"
 462                                                 bang git config --add "svn-remote.svnnew.$sname" "${gitsvnprefixnew}$sval"
 463                                         esac
 464                                 done
 465                                 test $? -eq 0
 466                                 bang git config -f svn/.metadata svn-remote.svn.reposRoot "$gitsvnurl"
 467                                 bang git config --remove-section svn-remote.svn
 468                                 bang git config --rename-section svn-remote.svnnew svn-remote.svn
 469                         fi
 470                         bang config_set svnurl "$svnurl"
 471                 fi
 472                 # remove any stale *.lock files greater than 1 hour old in case
 473                 # git-svn was killed on the last update because it took too long
 474                 find -L svn -type f -name '*.lock' -mmin +60 -exec rm -f '{}' + 2>/dev/null || :
 475                 GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1
 476                 export GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 477                 unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 478                 saveconfig="$GIT_CONFIG_PARAMETERS"
 479                 git_add_config 'gc.auto=1'
 480                 git_add_config 'gc.autoPackLimit=1'
 481                 GIT_DIR=. bang git_ulimit svn fetch --log-window-size=$var_log_window_size --username="$svnuser" --quiet <"$mtlinesfile"
 482                 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 483                 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 484                 unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 485                 unset GIT_CONFIG_PARAMETERS
 486                 [ -z "$saveconfig" ] || {
 487                         GIT_CONFIG_PARAMETERS="$saveconfig"
 488                         export GIT_CONFIG_PARAMETERS
 489                 }
 490                 # git svn does not preserve group permissions in the svn subdirectory
 491                 chmod -R ug+rw,o+r svn
 492                 # git svn also leaves behind ref turds that end with @nnn
 493                 # We get rid of them now
 494                 git for-each-ref --format='%(refname)' |
 495                 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
 496                 git_updateref_stdin
 497                 unset GIT_ASKPASS_PASSWORD
 498                 ;;
 499         darcs://* | darcs+http://* | darcs+https://*)
 500                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 0; }
 501                 case "$url" in
 502                         darcs://*) darcsurl="http://${url#darcs://}";;
 503                         *) darcsurl="${url#darcs+}";;
 504                 esac
 505                 # remove any stale lock files greater than 1 hour old in case
 506                 # darcs_fast_export was killed on the last update because it took too long
 507                 find -L *.darcs -maxdepth 2 -type f -name 'lock' -mmin +60 -exec rm -f '{}' + 2>/dev/null || :
 508                 bang git_darcs_fetch "$darcsurl"
 509                 ;;
 510         bzr://*)
 511                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 0; }
 512                 bzrurl="${url#bzr://}"
 513                 bang git_bzr_fetch "$bzrurl"
 514                 ;;
 515         hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
 516                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 0; }
 517                 # We just remove hg+ here, so hg+http://... becomes http://...
 518                 hgurl="${url#hg+}"
 519                 # Fetch any new updates
 520                 bang hg -R "$(pwd)/repo.hg" pull
 521                 # Do the fast-export | fast-import
 522                 bang git_hg_fetch
 523                 ;;
 524         *)
 525                 [ "$url" = "$(git config --get remote.origin.url || :)" ] || bang config_set_raw remote.origin.url "$url"
 526                 pruneopt=--prune
 527                 [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
 528                 if ! is_gfi_mirror_url "$url"; then
 529                         lastwasclean=
 530                         [ "$(git config --bool girocco.lastupdateclean 2>/dev/null || :)" != "true" ] || lastwasclean=1
 531                         nextisclean=
 532                         [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" != "true" ] || nextisclean=1
 533                         if [ "$nextisclean" != "$lastwasclean" ]; then
 534                                 if [ -n "$nextisclean" ]; then
 535                                         git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
 536                                         git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
 537                                         git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
 538                                         git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
 539                                         git config --add remote.origin.fetch "+refs/replace/*:refs/replace/*"
 540                                 else
 541                                         git config --replace-all remote.origin.fetch "+refs/*:refs/*"
 542                                 fi
 543                         fi
 544                         if ! create_quarantine; then
 545                                 bang echo ""
 546                                 bang echo "unable to create fetch quarantine area"
 547                                 bang_cmd="create_quarantine"
 548                                 bang_errcode=1
 549                                 bang_failed
 550                         fi
 551                 fi
 552                 # remember the starting time so we can easily detect new packs for fast-import mirrors
 553                 # we sleep for 1 second after creating .gfipack to make sure all packs are newer
 554                 if is_gfi_mirror_url "$url" && [ ! -e .gfipack ]; then
 555                         rm -f .gfipack
 556                         >.gfipack
 557                         sleep 1
 558                 fi
 559                 fetcharg="default"
 560                 git config remotes.default >/dev/null 2>&1 || fetcharg="--all"
 561                 GIT_DIR=.
 562                 [ -z "$use_quarantine" ] || GIT_DIR="$incoming_fetch"
 563                 fetchcmd="git_ulimit fetch"
 564                 [ "$show_progress" != "0" ] || fetchcmd="git_ulimit fetch -q"
 565                 if [ -n "$var_have_git_171" ] && [ "${show_progress:-0}" != "0" ]; then
 566                         # git fetch learned --progress in v1.7.1
 567                         case "$show_progress" in
 568                         [2-9]*|1[0-9]*)
 569                                 # full volume progress with all the spammy noise
 570                                 fetchcmd="git_ulimit fetch --progress"
 571                                 ;;
 572                         *)
 573                                 # a kinder, gentler progress that doesn't leave one
 574                                 # covered all over in exploded bits of spam afterwards
 575                                 fetchcmd="git_fetch_q_progress"
 576                                 ;;
 577                         esac
 578                 fi
 579                 # It's possible for a fetch to actually do something while still returning
 580                 # a non-zero result (perhaps some of the refs were updated but some were
 581                 # not -- a malicious Git-impersonation trying to set refs/heads/... refs
 582                 # to non-commit objects for example).
 583                 GIT_SSL_NO_VERIFY=1 bang_catch eval "$fetchcmd" $pruneopt --multiple "$fetcharg"
 584                 unset GIT_SSL_NO_VERIFY
 585                 # If we did fetch anything, don't treat it as an error, but do keep the log;
 586                 # otherwise invoke bang_failed as for a normal failure
 587                 if [ "${bang_errcode:-0}" != "0" ]; then
 588                         save_bang_errcode="$bang_errcode"
 589                         check_after_refs
 590                         if [ -n "$refschanged" ]; then
 591                                 keep_bang_log="git fetch${pruneopt:+ $pruneopt} --multiple $fetcharg"
 592                         else
 593                                 bang_cmd="git fetch${pruneopt:+ $pruneopt} --multiple $fetcharg"
 594                                 bang_errcode="$save_bang_errcode"
 595                                 bang_failed
 596                         fi
 597                 fi
 598                 if [ -n "$use_quarantine" ]; then
 599                         check_after_refs
 600                         unset GIT_DIR
 601                         if [ -n "$refschanged" ] && ! unquarantine_updates; then
 602                                 bang echo ""
 603                                 bang echo "unable to unquarantine fetched updates"
 604                                 bang_cmd="unquarantine_updates"
 605                                 bang_errcode=1
 606                                 bang_failed
 607                         fi
 608                 else
 609                         unset GIT_DIR
 610                 fi
 611                 if ! is_gfi_mirror_url "$url" && [ "$nextisclean" != "$lastwasclean" ]; then
 612                         if [ -n "$nextisclean" ]; then
 613                                 # We must manually purge the unclean refs now as even prune won't do it
 614                                 git for-each-ref --format='%(refname)' |
 615                                 LC_ALL=C sed \
 616                                         -e '/^refs\/heads\//d' \
 617                                         -e '/^refs\/tags\//d' \
 618                                         -e '/^refs\/notes\//d' \
 619                                         -e '/^refs\/top-bases\//d' \
 620                                         -e '/^refs\/replace\//d' \
 621                                         -e 's/^/delete /' |
 622                                 git_updateref_stdin
 623                         fi
 624                         git config --bool girocco.lastupdateclean ${nextisclean:-0}
 625                 fi
 626                 if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
 627                         find -L objects/pack -type f -newer .gfipack -name "pack-$octet20*.pack" -print >>gfi-packs
 628                         rm -f .gfipack
 629                 fi
 630                 # Check to see if we've lost our HEAD
 631                 if ! git rev-parse --verify HEAD >/dev/null 2>&1; then
 632                         git config --unset girocco.headok >/dev/null 2>&1 || :
 633                         # Try to get it back
 634                         check_and_set_head || :
 635                 fi
 636                 ;;
 637 esac
 638
 639 # The objects subdirectories permissions must be updated now.
 640 # In the case of a dumb http clone, the permissions will not be correct
 641 # (missing group write) despite the core.sharedrepository=2 setting!
 642 # The objects themselves seem to have the correct permissions.
 643 # This problem appears to have been fixed in the most recent git versions.
 644 perms=g+w
 645 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 646 chmod $perms $(find -L objects -maxdepth 1 -type d) 2>/dev/null || :
 647
 648 # We maintain the last refresh date in two places deliberately
 649 # so that it's available as part of the config data and also
 650 # as a standalone file timestamp that can be accessed without git.
 651 bang config_set lastrefresh "$(date "$datefmt")"
 652 { >.last_refresh; } 2>/dev/null || :
 653
 654 # Check to see if any refs changed
 655 check_after_refs
 656
 657 # Update server info if any refs changed (if they didn't packs shouldn't have either)
 658 [ -z "$refschanged" ] || bang git update-server-info
 659
 660 # Pack all refs if any changed to keep things as efficient as possible
 661 # Project mirror updates do not occur that often therefore this is a win
 662 # However, if pack-refs fails for some reason, we can just ignore and continue
 663 # The "--prune" option is the default since v1.5.0 but it serves as "documentation" here
 664 [ -z "$refschanged" ] || git pack-refs --all --prune || :
 665
 666 # Force a mini-gc if $Girocco::Config::delay_gfi_redelta is false and there's
 667 # at least one gfi pack present now
 668 if [ -z "$cfg_delay_gfi_redelta" ] && ! [ -e .needsgc ] &&
 669    [ -f gfi-packs ] && [ -s gfi-packs ] && is_gfi_mirror_url "$url"; then
 670         >.needsgc
 671 fi
 672
 673 # Activate a mini-gc if needed
 674 check_and_set_needsgc
 675
 676 # Look at which refs changed and trigger ref-change for these
 677 sockpath="$cfg_chroot/etc/taskd.socket"
 678 if [ -n "$refschanged" ]; then
 679         bang config_set lastreceive "$(date '+%a, %d %b %Y %T %z')"
 680         # We always use UTC for the log timestamp so that chroot and non-chroot match up.
 681         # We don't have to worry about multiple log files since only one update runs
 682         lognamets="$(TZ=UTC date '+%Y%m%d_%H%M%S')"
 683         loghhmmss="${lognamets##*_}"
 684         logname="reflogs/${lognamets%%_*}"
 685         # We freshen the mod time to now on any old or new ref that is a loose object
 686         # For old refs we do it so we will be able to keep them around for 1 day
 687         # For new refs we do it in case we are about to run gc and the new ref
 688         # actually points to an oldish loose object that had been unreachable
 689         # We probably do not need to do it for new refs as Git tries to do that,
 690         # but since we're already doing it for old refs (which Git does not do),
 691         # it's almost no extra work for new refs, just in case.
 692         {
 693                 echo "ref-changes %$proj% $proj"
 694                 LC_ALL=C join .refs-before .refs-after |
 695                 LC_ALL=C sed -e '/^[^ ][^ ]* \([^ ][^ ]*\) \1$/d' |
 696                         while read ref old new; do
 697                                 echo "$loghhmmss $old $new $ref" >&3
 698                                 freshen_loose_objects "$old" "$new"
 699                                 echo "$old $new $ref"
 700                         done
 701                 LC_ALL=C join -v 1 .refs-before .refs-after |
 702                         while read ref old; do
 703                                 echo "$loghhmmss $old 0000000000000000000000000000000000000000 $ref" >&3
 704                                 freshen_loose_objects "$old"
 705                                 echo "$old 0000000000000000000000000000000000000000 $ref"
 706                         done
 707                 LC_ALL=C join -v 2 .refs-before .refs-after |
 708                         while read ref new; do
 709                                 echo "$loghhmmss 0000000000000000000000000000000000000000 $new $ref" >&3
 710                                 freshen_loose_objects "$new"
 711                                 echo "0000000000000000000000000000000000000000 $new $ref"
 712                         done
 713                 git for-each-ref --format='%(objectname) %(objectname) %(refname)' refs/heads
 714                 echo "done ref-changes %$proj% $proj"
 715         } >.refs-temp 3>>"$logname"
 716         if [ -S "$sockpath" ]; then
 717                         trap ':' PIPE
 718                         nc_openbsd -w 15 -U "$sockpath" <.refs-temp || :
 719                         trap - PIPE
 720         fi
 721         bang config_set lastchange "$(date '+%a, %d %b %Y %T %z')"
 722         bang_eval "git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 723                 --count=1 refs/heads >info/lastactivity"
 724         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 725         rm -f .delaygc .allowgc
 726         if
 727                 [ "${cfg_autogchack:-0}" != "0" ] &&
 728                 [ "$(git config --get --bool girocco.autogchack 2>/dev/null)" != "false" ]
 729         then
 730                 mv -f .refs-after .refs-last
 731         fi
 732 fi
 733
 734 # If the repository does not yet have a valid HEAD symref try to set one
 735 # If an empty repository was cloned and then later becomes unempty you just
 736 # lose out on the fancy "symref=HEAD:" logic and get this version instead
 737 check_and_set_head || :
 738
 739 rm -f .refs-before .refs-after .refs-temp FETCH_HEAD
 740
 741 if is_banged; then
 742         [ -z "$mailaddrs" ] || ! was_banged_message_sent ||
 743         {
 744                 echo "$proj update succeeded - failure recovery"
 745                 echo "this status message may be disabled on the project admin page"
 746         } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj update succeeded" "$mailaddrs" || :
 747         bang_reset
 748 fi
 749
 750 if [ -n "$keep_bang_log" ] && [ -s "$bang_log" ]; then
 751         cat "$bang_log" >.banglog
 752         echo "" >>.banglog
 753         echo "$keep_bang_log failed with error code $save_bang_errcode" >>.banglog
 754 fi
 755
 756 progress "- [$proj] update ($(date))"