jobd/update.sh

   1 #!/bin/sh
   2
   3 . @basedir@/shlib.sh
   4 . @basedir@/jobd/updategc-util-functions.sh
   5
   6 set -e
   7
   8 if [ $# -ne 1 ]; then
   9         echo "Usage: update.sh projname" >&2
  10         exit 1
  11 fi
  12
  13 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
  14 datefmt='+%a, %d %b %Y %T %z'
  15
  16 git_fetch_q_progress() {
  17         if [ "${cfg_max_file_size512:-0}" != "0" ]; then
  18                 GIT_BIN="'$cfg_basedir/bin/ulimit512' -i -f '$cfg_max_file_size512' -- '$cfg_git_bin'" &&
  19                 export GIT_BIN
  20         fi
  21         PATH="$var_git_exec_path:$cfg_basedir/bin:$PATH" @basedir@/jobd/git-fetch-q-progress.sh "$@"
  22 }
  23
  24 # freshen_loose_objects full-sha ...
  25 # if "$n" is a loose object, set its modification time to now
  26 # otherwise silently do nothing with no error.  To facilitate conversion
  27 # of mirror projects to push projects we also add group write  permission.
  28 freshen_loose_objects() {
  29         _list=
  30         for _sha; do
  31                 _fn="${_sha#??}"
  32                 _shard="${_sha%$_fn}"
  33                 _list="$_list objects/$_shard/$_fn"
  34         done
  35         if [ -n "$_list" ]; then
  36                 chmod ug+w $_list 2>/dev/null || :
  37                 touch -c $_list 2>/dev/null || :
  38         fi
  39 }
  40
  41 # darcs fast-export | git fast-import with error handling
  42 git_darcs_fetch() (
  43         set_utf8_locale
  44         _err1=
  45         _err2=
  46         exec 3>&1
  47         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  48         $(
  49                 exec 4>&3 3>&1 1>&4 4>&-
  50                 {
  51                         _e1=0
  52                         "$cfg_basedir"/bin/darcs-fast-export \
  53                                 --export-marks="$(pwd)/dfe-marks" \
  54                                 --import-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
  55                         echo $_e1 >&3
  56                 } |
  57                 {
  58                         _e2=0
  59                         git_ulimit fast-import \
  60                                 --export-marks="$(pwd)/gfi-marks" \
  61                                 --export-pack-edges="$(pwd)/gfi-packs" \
  62                                 --import-marks="$(pwd)/gfi-marks" \
  63                                 --force 3>&- || _e2=$?
  64                         echo $_e2 >&3
  65                 }
  66         )
  67         EOT
  68         exec 3>&-
  69         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
  70         return $?
  71 )
  72
  73 # bzr fast-export | git fast-import with error handling
  74 git_bzr_fetch() (
  75         set_utf8_locale
  76         BZR_LOG=/dev/null
  77         export BZR_LOG
  78         _err1=
  79         _err2=
  80         exec 3>&1
  81         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
  82         $(
  83                 exec 4>&3 3>&1 1>&4 4>&-
  84                 {
  85                         _e1=0
  86                         bzr fast-export --plain \
  87                                 --export-marks="$(pwd)/bfe-marks" \
  88                                 --import-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
  89                         echo $_e1 >&3
  90                 } |
  91                 {
  92                         _e2=0
  93                         git_ulimit fast-import \
  94                                 --export-marks="$(pwd)/gfi-marks" \
  95                                 --export-pack-edges="$(pwd)/gfi-packs" \
  96                                 --import-marks="$(pwd)/gfi-marks" \
  97                                 --force 3>&- || _e2=$?
  98                         echo $_e2 >&3
  99                 }
 100         )
 101         EOT
 102         exec 3>&-
 103         [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
 104         return $?
 105 )
 106
 107 # On return a "$lockf" will have been created that must be removed when gc is done
 108 lock_update() {
 109         v_lock_file _lockresult "update.pid" || {
 110                 echo >&2 "[$proj] $_lockresult"
 111                 exit 1
 112         }
 113         lockf="$_lockresult"
 114 }
 115
 116 # output all loose object ids, one per line, to stdout INCLUDING "/" shard separator
 117 # look in "$1" (default "objects")
 118 list_loose_sharded_objects() (
 119         cd "${1:-objects}" || return 1
 120         objdirs="$(echo $octet)"
 121         [ "$objdirs" != "$octet" ] || return 0
 122         find -L $objdirs -mindepth 1 -maxdepth 1 -type f -name "$octet19*" -print
 123 )
 124
 125 # Migrate any and all objects/packs/ref-changes from $incoming_fetch/$incoming_objs
 126 # Does NOT do anything with reflogs, those should already be taken care of elsewhere
 127 unquarantine_updates() {
 128         # just make sure everthing is copacetic first
 129         [ -n "$incoming_fetch" ] && [ -d "$incoming_fetch" ] &&
 130         [ -n "$incoming_objs" ] && [ -d "$incoming_objs" ] || {
 131                 echo >&2 "[$proj] unquarantine failed"
 132                 exit 1
 133         }
 134         _ifd="$(git --git-dir="$incoming_fetch" rev-parse --git-dir 2>/dev/null)" || :
 135         [ -z "$_ifd" ] || _ifd="$(cd "$_ifd" && pwd -P)"
 136         [ -n "$_ifd" ] && [ "$_ifd" = "$incoming_fetch" ] || {
 137                 echo >&2 "[$proj] unquarantine failed"
 138                 exit 1
 139         }
 140
 141         # both $incoming_fetch and . must have all their refs packed
 142         git --git-dir="$incoming_fetch" pack-refs --all --prune
 143         git --git-dir=. pack-refs --all --prune
 144
 145         # now every loose object and pack must be migrated out of quarantine
 146         _objd="$(cd "$PWD/objects" && pwd -P)"
 147         # packs first
 148         if [ -d "$incoming_objs/pack" ]; then
 149                 if [ ! -d "$_objd/pack" ]; then
 150                         mkdir -p "$_objd/pack"
 151                         chmod "$dperms" "$_objd/pack" >/dev/null 2>&1 || :
 152                 fi
 153                 while read -r _pckf && [ -n "${_pckf%.pack}" ]; do
 154                         _pckf="${_pckf%.pack}"
 155                         rename_pack "$_pckf" "$_objd/pack/${_pckf##*/}"
 156                         chmod "$fperms" "$_objd/pack/${_pckf##*/}".?* >/dev/null 2>&1 || :
 157                 done <<LIST_PACKS
 158 $(list_packs --exclude-no-idx "$incoming_objs/pack")
 159 LIST_PACKS
 160         fi
 161         # now loose objects
 162         # (use a subshell for loose objects as there could potentially be many
 163         # whereas there will normally be just one pack)
 164         list_loose_sharded_objects "$incoming_objs" |
 165         while read -r _objf && [ -n "$_objf" ] && [ "${#_objf}" -ge 41 ]; do
 166                 if [ ! -d "$_objd/${_objf%%/*}" ]; then
 167                         mkdir -p "$_objd/${_objf%%/*}"
 168                         chmod "$dperms" "$_objd/${_objf%%/*}" >/dev/null 2>&1 || :
 169                 fi
 170                 ln "$incoming_objs/$_objf" "$_objd/$_objf" >/dev/null 2>&1 ||
 171                 dupe_file "$incoming_objs/$_objf" "$_objd/$_objf" "tmp_obj_" >/dev/null 2>&1 ||
 172                 [ -f "$_objd/$_objf" ] || {
 173                         echo >&2 "[$proj] unable to unquarantine object $_objf"
 174                         exit 1
 175                 }
 176                 chmod "$fperms" "$_objd/$_objf" >/dev/null 2>&1 || :
 177                 rm -f "$incoming_objs/$_objf"
 178         done || exit 1
 179
 180         # now the refs
 181         # simply replace the packed-refs file
 182         # but do it atomically and make sure it's on the same file system first
 183         rm -f "$PWD/packed-refs.$$"
 184         cat "$incoming_fetch/packed-refs" >"$PWD/packed-refs.$$"
 185         mv -f "$PWD/packed-refs.$$" "$PWD/packed-refs"
 186         rm -f "$PWD/packed-refs.$$"
 187 }
 188
 189 # Create a "quarantine" area to fetch into
 190 # This is set up similarly to the way the "repack" directory is set
 191 # up for gc in that it's a subdirectory that's a whole "git" directory
 192 # but it uses the existing objects directory as an alternate and its
 193 # own objects subdirectory is a symlink to a subdirectory of the real
 194 # objects directory (to guarantee that packs/objects can be moved rather
 195 # than copied).  It starts out with a copy of all of the project's refs.
 196 # A successful fetch will "unquarantine" fetched objects/packs + ref changes
 197 create_quarantine() {
 198         incoming_objs="$(mktemp -d "$PWD/objects/incoming-XXXXXX")"
 199         incoming_objs="$(cd "$incoming_objs" && pwd -P)"
 200         chmod "$dperms" "$incoming_objs"
 201         mkdir "$incoming_objs/pack"
 202         mkdir "$incoming_objs/info"
 203         printf '%s\n' "$PWD/objects" >"$incoming_objs/info/alternates"
 204         incoming_fetch="$(mktemp -d "$PWD/incoming-XXXXXX")"
 205         incoming_fetch="$(cd "$incoming_fetch" && pwd -P)"
 206         chmod "$dperms" "$incoming_fetch"
 207         ln -s "$incoming_objs" "$incoming_fetch/objects"
 208         mkdir "$incoming_fetch/refs"
 209         ln -s "$PWD/config" "$incoming_fetch/config"
 210         git for-each-ref --format='%(objectname) %(refname)' >"$incoming_fetch/packed-refs"
 211         cat HEAD >"$incoming_fetch/HEAD"
 212         # Make sure the incoming packed-refs file is properly peeled
 213         git --git-dir="$incoming_fetch" pack-refs --all --prune
 214         # link to svn if it exists
 215         [ ! -d svn ] || ln -s "$PWD/svn" "$incoming_fetch/svn"
 216         use_quarantine=1
 217 }
 218
 219 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 0; }
 220
 221 if [ "$cfg_permission_control" != "Hooks" ]; then
 222         umask 002
 223         fperms=0664
 224         dperms=2775
 225 else
 226         umask 000
 227         fperms=0666
 228         dperms=2777
 229 fi
 230 clean_git_env
 231
 232 proj="${1%.git}"
 233 cd "$cfg_reporoot/$proj.git"
 234
 235 # Activate a mini-gc if needed
 236 # We do this here as well as after a successful fetch so that if we're stuck
 237 # in a fetch loop where fetches are succeeding in fetching new packs but the
 238 # ref update is failing for some reason (perhaps a non-commit under refs/heads)
 239 # and a previous invokation therefore had a "bang" exit then we will still
 240 # get the .needsgc flag set in a timely fashion to avoid excess pack build up.
 241 check_and_set_needsgc
 242
 243 use_quarantine=
 244 bang_log=
 245 incoming_fetch=
 246 incoming_objs=
 247 lockf=
 248 cleanup_exit() {
 249         ec=$?
 250         if [ $ec != 0 ]; then
 251                 echo "update failed dir: $PWD" >&2
 252         fi
 253         [ -z "$incoming_fetch" ] || rm -rf "$incoming_fetch"
 254         [ -z "$incoming_objs" ] || rm -rf "$incoming_objs"
 255         [ -z "$bang_log" ] || rm -f "$bang_log"
 256         [ -z "$lockf" ] || rm -f "$lockf"
 257 }
 258 trap 'cleanup_exit' EXIT
 259 trap 'exit 129' HUP
 260 trap 'exit 130' INT
 261 trap 'exit 131' QUIT
 262 trap 'exit 134' ABRT
 263 trap 'exit 141' PIPE
 264 trap 'exit 142' ALRM
 265 trap 'exit 143' TERM
 266
 267 if [ "${force_update:-0}" = "0" ] && check_interval lastrefresh $cfg_min_mirror_interval; then
 268         progress "= [$proj] update skip (last at $(config_get lastrefresh))"
 269         exit 0
 270 fi
 271 if [ -e .nofetch ]; then
 272         progress "x [$proj] update disabled (.nofetch exists)"
 273         exit 0
 274 fi
 275 lock_update
 276 progress "+ [$proj] update ($(date))"
 277
 278 # Any pre-existing FETCH_HEAD from a previous fetch, failed or not, is garbage
 279 rm -f FETCH_HEAD
 280
 281 # Remove any stale ref locks
 282 clear_stale_ref_locks
 283
 284 # Remove any stale incoming-* object quarantine directories that are
 285 # more than 12 hours old.  These are new with Git >= 2.11.0.
 286 # But we also create our own during the fetch process as Git's quarantine
 287 # only applies to incoming receive-pack which we imitate for our fetch.
 288 find -L . objects -maxdepth 1 -type d -name 'incoming-?*' -mmin +720 \
 289         -exec rm -rf '{}' + || :
 290
 291 # A previous failed update attempt can leave a huge tmp_pack_XXXXXX file behind.
 292 # Since no pushes are allowed to mirrors, we know that any such files that exist
 293 # at this point in time are garbage and can be safely deleted, we do not even
 294 # need to check how old they are.  A tmp_idx_XXXXXX file is also created during
 295 # the later stages of the fetch process, so we kill any of those as well.
 296 find -L objects/pack -maxdepth 1 -type f -name "tmp_pack_?*" -exec rm -f '{}' + || :
 297 find -L objects/pack -maxdepth 1 -type f -name "tmp_idx_?*" -exec rm -f '{}' + || :
 298
 299 # Make sure we have a reflogs subdirectory and abort the update if not
 300 # This should not count as a normal "bang" failure if unsuccessful
 301 [ -d reflogs ] || mkdir -p reflogs >/dev/null 2>&1 || :
 302 [ -d reflogs ]
 303
 304 keep_bang_log=
 305 do_check_after_refs=1
 306 bang_setup
 307 bang_action="update"
 308 bang_trap() {
 309         if [ -n "$1" ]; then
 310                 # Throttle retries
 311                 # Since gitweb shows the .last_refresh date, it's safe to update
 312                 # gitweb.lastrefresh to throttle the updates w/o corrupting the
 313                 # last refresh date display on the gitweb summary page
 314                 # It's therefore important that we do NOT touch .last_refresh here
 315                 config_set lastrefresh "$(date "$datefmt")"
 316         fi
 317 }
 318
 319 bang echo "Project: $proj"
 320 bang echo "   Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
 321 bang echo ""
 322 mail="$(config_get owner)" || :
 323 url="$(config_get baseurl)" || :
 324 case "$url" in *"       "*|*" "*|"")
 325         bang_eval 'echo "Bad mirror URL (\"$url\")"; ! :'
 326         exit 1
 327 esac
 328 bang echo "Mirroring from URL \"$url\""
 329 bang echo ""
 330 statusok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 331 mailaddrs=
 332 [ "$statusok" = "false" ] || [ -z "$mail" ] || mailaddrs="$mail"
 333 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 334 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
 335
 336 bang_eval "git for-each-ref --format '%(refname) %(objectname)' >.refs-temp"
 337 bang_eval "LC_ALL=C sort -b -k1,1 <.refs-temp >.refs-before"
 338
 339 check_after_refs() {
 340         [ -n "$do_check_after_refs" ] || return 0
 341         bang_eval "git for-each-ref --format '%(refname) %(objectname)' >.refs-temp"
 342         bang_eval "LC_ALL=C sort -b -k1,1 <.refs-temp >.refs-after"
 343         refschanged=
 344         cmp -s .refs-before .refs-after || refschanged=1
 345         do_check_after_refs=
 346 }
 347
 348 ! [ -e .delaygc ] || >.allowgc || :
 349
 350 # Make sure we don't get any unwanted loose objects
 351 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
 352 # tweak its configuration to prevent that
 353 git_add_config 'fetch.unpackLimit=1'
 354 # Note the git config documentation is wrong
 355 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
 356 git_add_config 'transfer.unpackLimit=1'
 357 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
 358 git_add_config 'fastimport.unpackLimit=0'
 359
 360 # remember the starting time so we can easily combine fetched loose objects
 361 # we sleep for 1 second after creating .needspack to make sure all objects are newer
 362 if ! [ -e .needspack ]; then
 363         rm -f .needspack
 364         >.needspack
 365         sleep 1
 366 fi
 367
 368 case "$url" in
 369         svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 370                 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 0; }
 371                 # Allow the username to be specified in the "svn-credential.svn.username"
 372                 # property and the password in the "svn-credential.svn.password" property
 373                 # Use an 'anonsvn' username by default as is commonly used for anonymous svn
 374                 # Default the password to the same as the username
 375                 # The password property will be ignored unless a username has been specified
 376                 if svnuser="$(git config --get svn-credential.svn.username)" && [ -n "$svnuser" ]; then
 377                         if ! svnpass="$(git config --get svn-credential.svn.password)"; then
 378                                 svnpass="$svnuser"
 379                         fi
 380                         url1="${url#*://}"
 381                         url1="${url1%%/*}"
 382                         case "$url1" in ?*"@"?*)
 383                                 urlsch="${url%%://*}"
 384                                 url="$urlsch://${url#*@}"
 385                         esac
 386                 else
 387                         # As a fallback, check in the URL, just in case
 388                         url1="${url#*://}"
 389                         url1="${url1%%/*}"
 390                         svnuser=
 391                         case "$url1" in ?*"@"?*)
 392                                 urlsch="${url%%://*}"
 393                                 url="$urlsch://${url#*@}"
 394                                 url1="${url1%%@*}"
 395                                 svnuser="${url1%%:*}"
 396                                 if [ -n "$svnuser" ]; then
 397                                         svnpass="$svnuser"
 398                                         case "$url1" in *":"*)
 399                                                 svnpass="${url1#*:}"
 400                                         esac
 401                                 fi
 402                         esac
 403                         if [ -z "$svnuser" ]; then
 404                                 svnuser="anonsvn"
 405                                 svnpass="anonsvn"
 406                         fi
 407                 fi
 408                 GIT_ASKPASS_PASSWORD="$svnpass"
 409                 export GIT_ASKPASS_PASSWORD
 410                 # Update the git svn url to match baseurl but be cognizant of any
 411                 # needed prefix changes.  See the comments in taskd/clone.sh about
 412                 # why we need to put up with a prefix in the first place.
 413                 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
 414                 svnurl="${svnurl%/}"
 415                 svnurlold="$(config_get svnurl)" || :
 416                 if [ "$svnurl" != "$svnurlold" ]; then
 417                         # We better already have an svn-remote.svn.fetch setting
 418                         bang test -n "$(git config --get-all svn-remote.svn.fetch)" || :
 419                         # the only way to truly know what the proper prefix is
 420                         # is to attempt a fresh git-svn init -s on the new url
 421                         rm -rf svn-new-url || :
 422                         # We require svn info to succeed on the URL otherwise it's
 423                         # simply not a valid URL and without using -s on the init it
 424                         # will not otherwise be tested until the fetch
 425                         bang eval 'svn --non-interactive --username "$svnuser" --password "$svnpass" info "$svnurl" >/dev/null'
 426                         bang mkdir svn-new-url
 427                         GIT_DIR=svn-new-url bang git init --bare --quiet
 428                         # We initially use -s for the init which will possibly shorten
 429                         # the URL.  However, the shortening can fail if a password is
 430                         # not required for the longer version but is for the shorter,
 431                         # so try again without -s if the -s version fails.
 432                         cmdstr='git svn init --username="$svnuser" --prefix "" -s "$svnurl" <"$mtlinesfile" >/dev/null 2>&1 || '
 433                         cmdstr="$cmdstr"'git svn init --username="$svnuser" --prefix "" "$svnurl" <"$mtlinesfile" >/dev/null 2>&1'
 434                         GIT_DIR=svn-new-url bang eval "$cmdstr"
 435                         gitsvnurl="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.url)" || :
 436                         gitsvnfetch="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.fetch)" || :
 437                         gitsvnprefixnew="${gitsvnfetch%%:*}"
 438                         gitsvnsuffixnew="${gitsvnprefixnew##*/}"
 439                         gitsvnprefixnew="${gitsvnprefixnew%$gitsvnsuffixnew}"
 440                         rm -rf svn-new-url || :
 441                         # Using GIT_DIR= with bang leaves it set to svn-new-url, so reset it to .
 442                         GIT_DIR=.
 443                         if [ "$gitsvnurl" != "$(git config --get svn-remote.svn.url || :)" ]; then
 444                                 # The url has been changed.
 445                                 # We must update the url and replace the prefix on all config items
 446                                 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | head -1)" || :
 447                                 gitsvnprefixold="${gitsvnfetch%%:*}"
 448                                 gitsvnsuffixold="${gitsvnprefixold##*/}"
 449                                 gitsvnprefixold="${gitsvnprefixold%$gitsvnsuffixold}"
 450                                 git config --remove-section 'svn-remote.svnnew' 2>/dev/null || :
 451                                 git config 'svn-remote.svnnew.url' "$gitsvnurl"
 452                                 git config --get-regexp '^svn-remote\.svn\.' |
 453                                 while read -r sname sval; do
 454                                         case "$sname" in
 455                                         svn-remote.svn.fetch|svn-remote.svn.branches|svn-remote.svn.tags)
 456                                                 sname="${sname#svn-remote.svn.}"
 457                                                 sval="${sval#$gitsvnprefixold}"
 458                                                 bang git config --add "svn-remote.svnnew.$sname" "${gitsvnprefixnew}$sval"
 459                                         esac
 460                                 done
 461                                 test $? -eq 0
 462                                 bang git config -f svn/.metadata svn-remote.svn.reposRoot "$gitsvnurl"
 463                                 bang git config --remove-section svn-remote.svn
 464                                 bang git config --rename-section svn-remote.svnnew svn-remote.svn
 465                         fi
 466                         bang config_set svnurl "$svnurl"
 467                 fi
 468                 # remove any stale *.lock files greater than 1 hour old in case
 469                 # git-svn was killed on the last update because it took too long
 470                 find -L svn -type f -name '*.lock' -mmin +60 -exec rm -f '{}' + 2>/dev/null || :
 471                 GIROCCO_DIVERT_GIT_SVN_AUTO_GC=1
 472                 export GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 473                 unset GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 474                 saveconfig="$GIT_CONFIG_PARAMETERS"
 475                 git_add_config 'gc.auto=1'
 476                 git_add_config 'gc.autoPackLimit=1'
 477                 GIT_DIR=. bang git_ulimit svn fetch --log-window-size=$var_log_window_size --username="$svnuser" --quiet <"$mtlinesfile"
 478                 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 479                 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 480                 unset GIROCCO_DIVERT_GIT_SVN_AUTO_GC
 481                 unset GIT_CONFIG_PARAMETERS
 482                 [ -z "$saveconfig" ] || {
 483                         GIT_CONFIG_PARAMETERS="$saveconfig"
 484                         export GIT_CONFIG_PARAMETERS
 485                 }
 486                 # git svn does not preserve group permissions in the svn subdirectory
 487                 chmod -R ug+rw,o+r svn
 488                 # git svn also leaves behind ref turds that end with @nnn
 489                 # We get rid of them now
 490                 git for-each-ref --format='%(refname)' |
 491                 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
 492                 git_updateref_stdin
 493                 unset GIT_ASKPASS_PASSWORD
 494                 ;;
 495         darcs://* | darcs+http://* | darcs+https://*)
 496                 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 0; }
 497                 case "$url" in
 498                         darcs://*) darcsurl="http://${url#darcs://}";;
 499                         *) darcsurl="${url#darcs+}";;
 500                 esac
 501                 # remove any stale lock files greater than 1 hour old in case
 502                 # darcs_fast_export was killed on the last update because it took too long
 503                 find -L *.darcs -maxdepth 2 -type f -name 'lock' -mmin +60 -exec rm -f '{}' + 2>/dev/null || :
 504                 bang git_darcs_fetch "$darcsurl"
 505                 ;;
 506         bzr://*)
 507                 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 0; }
 508                 bzrurl="${url#bzr://}"
 509                 bang git_bzr_fetch "$bzrurl"
 510                 ;;
 511         hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
 512                 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 0; }
 513                 # We just remove hg+ here, so hg+http://... becomes http://...
 514                 hgurl="${url#hg+}"
 515                 # Fetch any new updates
 516                 bang hg -R "$(pwd)/repo.hg" pull
 517                 # Do the fast-export | fast-import
 518                 bang git_hg_fetch
 519                 ;;
 520         *)
 521                 [ "$url" = "$(git config --get remote.origin.url || :)" ] || bang config_set_raw remote.origin.url "$url"
 522                 pruneopt=--prune
 523                 [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
 524                 if ! is_gfi_mirror_url "$url"; then
 525                         lastwasclean=
 526                         [ "$(git config --bool girocco.lastupdateclean 2>/dev/null || :)" != "true" ] || lastwasclean=1
 527                         nextisclean=
 528                         [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" != "true" ] || nextisclean=1
 529                         if [ "$nextisclean" != "$lastwasclean" ]; then
 530                                 if [ -n "$nextisclean" ]; then
 531                                         git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
 532                                         git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
 533                                         git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
 534                                         git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
 535                                 else
 536                                         git config --replace-all remote.origin.fetch "+refs/*:refs/*"
 537                                 fi
 538                         fi
 539                         if ! create_quarantine; then
 540                                 bang echo ""
 541                                 bang echo "unable to create fetch quarantine area"
 542                                 bang_cmd="create_quarantine"
 543                                 bang_errcode=1
 544                                 bang_failed
 545                         fi
 546                 fi
 547                 # remember the starting time so we can easily detect new packs for fast-import mirrors
 548                 # we sleep for 1 second after creating .gfipack to make sure all packs are newer
 549                 if is_gfi_mirror_url "$url" && [ ! -e .gfipack ]; then
 550                         rm -f .gfipack
 551                         >.gfipack
 552                         sleep 1
 553                 fi
 554                 fetcharg="default"
 555                 git config remotes.default >/dev/null 2>&1 || fetcharg="--all"
 556                 GIT_DIR=.
 557                 [ -z "$use_quarantine" ] || GIT_DIR="$incoming_fetch"
 558                 fetchcmd="git_ulimit fetch"
 559                 [ "$show_progress" != "0" ] || fetchcmd="git_ulimit fetch -q"
 560                 if [ -n "$var_have_git_171" ] && [ "${show_progress:-0}" != "0" ]; then
 561                         # git fetch learned --progress in v1.7.1
 562                         case "$show_progress" in
 563                         [2-9]*|1[0-9]*)
 564                                 # full volume progress with all the spammy noise
 565                                 fetchcmd="git_ulimit fetch --progress"
 566                                 ;;
 567                         *)
 568                                 # a kinder, gentler progress that doesn't leave one
 569                                 # covered all over in exploded bits of spam afterwards
 570                                 fetchcmd="git_fetch_q_progress"
 571                                 ;;
 572                         esac
 573                 fi
 574                 # It's possible for a fetch to actually do something while still returning
 575                 # a non-zero result (perhaps some of the refs were updated but some were
 576                 # not -- a malicious Git-impersonation trying to set refs/heads/... refs
 577                 # to non-commit objects for example).
 578                 GIT_SSL_NO_VERIFY=1 bang_catch eval "$fetchcmd" $pruneopt --multiple "$fetcharg"
 579                 unset GIT_SSL_NO_VERIFY
 580                 # If we did fetch anything, don't treat it as an error, but do keep the log;
 581                 # otherwise invoke bang_failed as for a normal failure
 582                 if [ "${bang_errcode:-0}" != "0" ]; then
 583                         save_bang_errcode="$bang_errcode"
 584                         check_after_refs
 585                         if [ -n "$refschanged" ]; then
 586                                 keep_bang_log="git fetch${pruneopt:+ $pruneopt} --multiple $fetcharg"
 587                         else
 588                                 bang_cmd="git fetch${pruneopt:+ $pruneopt} --multiple $fetcharg"
 589                                 bang_errcode="$save_bang_errcode"
 590                                 bang_failed
 591                         fi
 592                 fi
 593                 if [ -n "$use_quarantine" ]; then
 594                         check_after_refs
 595                         unset GIT_DIR
 596                         if [ -n "$refschanged" ] && ! unquarantine_updates; then
 597                                 bang echo ""
 598                                 bang echo "unable to unquarantine fetched updates"
 599                                 bang_cmd="unquarantine_updates"
 600                                 bang_errcode=1
 601                                 bang_failed
 602                         fi
 603                 else
 604                         unset GIT_DIR
 605                 fi
 606                 if ! is_gfi_mirror_url "$url" && [ "$nextisclean" != "$lastwasclean" ]; then
 607                         if [ -n "$nextisclean" ]; then
 608                                 # We must manually purge the unclean refs now as even prune won't do it
 609                                 git for-each-ref --format='%(refname)' |
 610                                 LC_ALL=C sed \
 611                                         -e '/^refs\/heads\//d' \
 612                                         -e '/^refs\/tags\//d' \
 613                                         -e '/^refs\/notes\//d' \
 614                                         -e '/^refs\/top-bases\//d' \
 615                                         -e 's/^/delete /' |
 616                                 git_updateref_stdin
 617                         fi
 618                         git config --bool girocco.lastupdateclean ${nextisclean:-0}
 619                 fi
 620                 if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
 621                         find -L objects/pack -type f -newer .gfipack -name "pack-$octet20*.pack" -print >>gfi-packs
 622                         rm -f .gfipack
 623                 fi
 624                 ;;
 625 esac
 626
 627 # The objects subdirectories permissions must be updated now.
 628 # In the case of a dumb http clone, the permissions will not be correct
 629 # (missing group write) despite the core.sharedrepository=1 setting!
 630 # The objects themselves seem to have the correct permissions.
 631 # This problem appears to have been fixed in the most recent git versions.
 632 perms=g+w
 633 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
 634 chmod $perms $(find -L objects -maxdepth 1 -type d) 2>/dev/null || :
 635
 636 # We maintain the last refresh date in two places deliberately
 637 # so that it's available as part of the config data and also
 638 # as a standalone file timestamp that can be accessed without git.
 639 bang config_set lastrefresh "$(date "$datefmt")"
 640 { >.last_refresh; } 2>/dev/null || :
 641
 642 # Check to see if any refs changed
 643 check_after_refs
 644
 645 # Update server info if any refs changed (if they didn't packs shouldn't have either)
 646 [ -z "$refschanged" ] || bang git update-server-info
 647
 648 # Pack all refs if any changed to keep things as efficient as possible
 649 # Project mirror updates do not occur that often therefore this is a win
 650 # However, if pack-refs fails for some reason, we can just ignore and continue
 651 # The "--prune" option is the default since v1.5.0 but it serves as "documentation" here
 652 [ -z "$refschanged" ] || git pack-refs --all --prune || :
 653
 654 # Force a mini-gc if $Girocco::Config::delay_gfi_redelta is false and there's
 655 # at least one gfi pack present now
 656 if [ -z "$cfg_delay_gfi_redelta" ] && ! [ -e .needsgc ] &&
 657    [ -f gfi-packs ] && [ -s gfi-packs ] && is_gfi_mirror_url "$url"; then
 658         >.needsgc
 659 fi
 660
 661 # Activate a mini-gc if needed
 662 check_and_set_needsgc
 663
 664 # Look at which refs changed and trigger ref-change for these
 665 sockpath="$cfg_chroot/etc/taskd.socket"
 666 if [ -n "$refschanged" ]; then
 667         bang config_set lastreceive "$(date '+%a, %d %b %Y %T %z')"
 668         # We always use UTC for the log timestamp so that chroot and non-chroot match up.
 669         # We don't have to worry about multiple log files since only one update runs
 670         lognamets="$(TZ=UTC date '+%Y%m%d_%H%M%S')"
 671         loghhmmss="${lognamets##*_}"
 672         logname="reflogs/${lognamets%%_*}"
 673         # We freshen the mod time to now on any old or new ref that is a loose object
 674         # For old refs we do it so we will be able to keep them around for 1 day
 675         # For new refs we do it in case we are about to run gc and the new ref
 676         # actually points to an oldish loose object that had been unreachable
 677         # We probably do not need to do it for new refs as Git tries to do that,
 678         # but since we're already doing it for old refs (which Git does not do),
 679         # it's almost no extra work for new refs, just in case.
 680         {
 681                 echo "ref-changes %$proj% $proj"
 682                 LC_ALL=C join .refs-before .refs-after |
 683                 LC_ALL=C sed -e '/^[^ ][^ ]* \([^ ][^ ]*\) \1$/d' |
 684                         while read ref old new; do
 685                                 echo "$loghhmmss $old $new $ref" >&3
 686                                 freshen_loose_objects "$old" "$new"
 687                                 echo "$old $new $ref"
 688                         done
 689                 LC_ALL=C join -v 1 .refs-before .refs-after |
 690                         while read ref old; do
 691                                 echo "$loghhmmss $old 0000000000000000000000000000000000000000 $ref" >&3
 692                                 freshen_loose_objects "$old"
 693                                 echo "$old 0000000000000000000000000000000000000000 $ref"
 694                         done
 695                 LC_ALL=C join -v 2 .refs-before .refs-after |
 696                         while read ref new; do
 697                                 echo "$loghhmmss 0000000000000000000000000000000000000000 $new $ref" >&3
 698                                 freshen_loose_objects "$new"
 699                                 echo "0000000000000000000000000000000000000000 $new $ref"
 700                         done
 701                 git for-each-ref --format='%(objectname) %(objectname) %(refname)' refs/heads
 702                 echo "done ref-changes %$proj% $proj"
 703         } >.refs-temp 3>>"$logname"
 704         if [ -S "$sockpath" ]; then
 705                         trap ':' PIPE
 706                         nc_openbsd -w 15 -U "$sockpath" <.refs-temp || :
 707                         trap - PIPE
 708         fi
 709         bang config_set lastchange "$(date '+%a, %d %b %Y %T %z')"
 710         bang_eval "git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
 711                 --count=1 refs/heads >info/lastactivity"
 712         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 713         rm -f .delaygc .allowgc
 714         if
 715                 [ "${cfg_autogchack:-0}" != "0" ] &&
 716                 [ "$(git config --get --bool girocco.autogchack 2>/dev/null)" != "false" ]
 717         then
 718                 mv -f .refs-after .refs-last
 719         fi
 720 fi
 721
 722 # If the repository does not yet have a valid HEAD symref try to set one
 723 # If an empty repository was cloned and then later becomes unempty you just
 724 # lose out on the fancy "symref=HEAD:" logic and get this version instead
 725 check_and_set_head || :
 726
 727 rm -f .refs-before .refs-after .refs-temp FETCH_HEAD
 728
 729 if is_banged; then
 730         [ -z "$mailaddrs" ] || ! was_banged_message_sent ||
 731         {
 732                 echo "$proj update succeeded - failure recovery"
 733                 echo "this status message may be disabled on the project admin page"
 734         } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj update succeeded" "$mailaddrs" || :
 735         bang_reset
 736 fi
 737
 738 if [ -n "$keep_bang_log" ] && [ -s "$bang_log" ]; then
 739         cat "$bang_log" >.banglog
 740         echo "" >>.banglog
 741         echo "$keep_bang_log failed with error code $save_bang_errcode" >>.banglog
 742 fi
 743
 744 progress "- [$proj] update ($(date))"