toolbox/reports/project-fsck-status.sh

   1 #!/bin/sh
   2
   3 # Report on project repository status.
   4 # Output can be sent as email to admin with -m
   5 # Automatically runs with nice and ionice (if available)
   6
   7 set -e
   8
   9 datefmt='%Y-%m-%d %H:%M:%S %z'
  10 startdate="$(date "+$datefmt")"
  11
  12 . @basedir@/shlib.sh
  13 PATH="$cfg_basedir/bin:${PATH:-$(/usr/bin/getconf PATH)}"
  14 export PATH
  15
  16 USAGE="${0##*/} [-hmEW] [--full] [<project>...]"
  17 HELP="
  18 NAME
  19        ${0##*/} - report on \"health\" of Girocco repositories
  20
  21 SYNOPSIS
  22        $USAGE
  23
  24 DESCRIPTION
  25        The ${0##*/} script checks all Girocco repositories for
  26        consistency and includes a count of empty repositories as well.
  27
  28        The results can automatically be mailed to the Girocco admin or
  29        simply displayed on STDOUT or optionally suppressed if there are
  30        no errors or warnings.
  31
  32        Note that this utility currently provides absolutely no progress
  33        reporting even when run on a terminal so there will be no output
  34        whatsoever until all projects have been checked.
  35
  36        The \"checking\" runs using nice and, if available, ionice.
  37
  38        Projects not listed in \$chroot/etc/group will not be checked.
  39
  40 OPTIONS
  41        -h     show this help
  42
  43        -m     e-mail results to the \$Girocco::Config::admin address.
  44               This option suppresses output to STDOUT.
  45
  46        -E     Suppress output (i.e. no e-mail will be sent if -m has been used)
  47               unless at least one error has been detected.  If this flag is
  48               used WITHOUT -W and warnings occur, output will still be
  49               suppressed.
  50
  51        -W     Suppress output unless at least one error or warning has been
  52               detected.  This flag overrides -E and will always produce output
  53               when only warnings are detected.
  54
  55        --full Run a full \"git fsck\" check on each repository.  Without this
  56               option a much, much, much (5x or more) faster check is done on
  57               each repository using \"git rev-list --objects --all\" that will
  58               verify all objects reachable from any ref are present but does
  59               not perform all the validation checks nor does it verify the
  60               integrity of 'blob' objects (just that they are present).
  61
  62               If the default Girocco 'transfer.fsckObjects=true' option has
  63               been left intact, use of the \"--full\" option should not
  64               normally be necessary.
  65
  66        --no-full
  67               Disables --full.  Present for completeness.
  68
  69        --reflog
  70               Check ref logs too.  Implied by \"--full\" but not by \"--no-full\".
  71
  72        --indexed-objects
  73               Check indexed objects too.  Implied by \"--full\" but not by
  74               \"--no-full\".  Requires Git version 2.2.0 or later if not using
  75               the \"--full\" option to do it.
  76
  77        --list Show a list of all projects checked in the order checked.
  78               This is the default if any specific projects are given.
  79
  80        --no-list
  81               Do NOT show a list of each project name checked.
  82               This is the default when all projects are being checked.
  83
  84        <project>
  85               Optionally one or more Girocco project names may be given to
  86               check only those projects.
  87
  88 TIPS
  89        Leave Girocco's default 'transfer.fsckObjects=true' setting alone and
  90        have a cron job run this utility in the default --no-full mode using the
  91        -m and -E options once a week to detect repository corruption.
  92
  93        The default --no-full mode will detect any fatal errors with all
  94        reachable 'tag', 'commit' and 'tree' objects.  The --no-full mode will
  95        detect missing 'blob' objects but not corrupt 'blob' objects.
  96
  97        The --no-full mode is an order of magnitude faster than --full mode and
  98        does not complain about non-fatal object errors.
  99 "
 100
 101 usage() {
 102         if [ "${1:-1}" = "1" ]; then
 103                 printf 'usage: %s\n' "$USAGE" >&2
 104                 return 0
 105         fi
 106         printf '%s\n' "$HELP"
 107 }
 108
 109 mailresult=
 110 minoutput=0 # 0 => always output, 1 => if warn or err, 2 => if err
 111 fullcheck=
 112 onlylist=
 113 shownames=
 114 reflog=
 115 indexedobj=
 116
 117 while [ $# -gt 0 ]; do
 118         if [ "${1#-[!-]?}" != "$1" ]; then
 119                 _rest="${1#-?}"
 120                 _next="${1%"$_rest"}"
 121                 shift
 122                 set -- "$_next" "-$_rest" "$@"
 123         fi
 124         case "$1" in
 125         "-?")
 126                 usage 1 2>&1
 127                 exit 0
 128                 ;;
 129         -h|--help)
 130                 usage 2
 131                 exit 0
 132                 ;;
 133         -m|--mail)
 134                 mailresult=1
 135                 ;;
 136         -E)
 137                 [ "${minoutput:-0}" != "0" ] || minoutput=2
 138                 ;;
 139         -W)
 140                 minoutput=1
 141                 ;;
 142         --full)
 143                 fullcheck=1
 144                 ;;
 145         --no-full)
 146                 fullcheck=
 147                 ;;
 148         --reflog)
 149                 reflog=1
 150                 ;;
 151         --indexed-objects)
 152                 indexedobj=1
 153                 ;;
 154         --list)
 155                 shownames=1
 156                 ;;
 157         --no-list)
 158                 shownames=0
 159                 ;;
 160         --)
 161                 shift
 162                 break
 163                 ;;
 164         -?*)
 165                 echo "${0##*/}: unknown option: $1" >&2
 166                 usage 1
 167                 exit 1
 168                 ;;
 169         *)
 170                 break
 171                 ;;
 172         esac
 173         shift
 174 done
 175
 176 [ -n "$shownames" ] || [ $# -eq 0 ] || shownames=1
 177
 178 while [ $# -gt 0 ]; do
 179         aproj="${1%.git}"
 180         shift
 181         if ! [ -d "$cfg_reporoot/$aproj.git" ]; then
 182                 echo "${0##*/}: fatal: no such dir: $cfg_reporoot/$aproj.git" >&2
 183                 exit 1
 184         fi
 185         if ! is_git_dir "$cfg_reporoot/$aproj.git"; then
 186                 echo "${0##*/}: fatal: not a git dir: $cfg_reporoot/$aproj.git" >&2
 187                 exit 1
 188         fi
 189         onlylist="${onlylist:+$onlylist }$aproj.git"
 190 done
 191
 192 hasnice=
 193 ! command -v nice >/dev/null || hasnice=1
 194 hasionice=
 195 ! command -v ionice >/dev/null || hasionice=1
 196
 197 nl='
 198 '
 199
 200 projlist="$(cut -d : -f 1 <"$cfg_chroot/etc/group")"
 201
 202 is_listed_proj() {
 203         echo "$projlist" | grep -q -e "^$1$"
 204 }
 205
 206 is_empty_proj() {
 207         # if packed-refs is empty and no files in refs then empty
 208         # we do NOT want to run any git command in case the repo is bad
 209         _pd="$cfg_reporoot/$1.git"
 210         if [ -f "$_pd/packed-refs" ] && [ -s "$_pd/packed-refs" ]; then
 211                 if [ $(LC_ALL=C sed -n '/^#/!p' <"$_pd/packed-refs" | LC_ALL=C wc -l) -gt 0 ]; then
 212                         return 1
 213                 fi
 214         fi
 215         test $(find -L "$_pd/refs" -type f -print 2>/dev/null | head -n 1 | LC_ALL=C wc -l) -eq 0
 216 }
 217
 218 get_check_proj() (
 219         cd "$cfg_reporoot/$1.git" || {
 220                 echo "no such directory: $cfg_reporoot/$1.git"
 221                 return 1
 222         }
 223         if [ -n "$fullcheck" ]; then
 224                 # use git fsck
 225                 # using --strict changes "zero-padded file modes" from a warning into an error
 226                 # which we do NOT want so we do NOT use --strict
 227                 cmd="git fsck"
 228                 [ -z "$var_have_git_1710" ] || cmd="$cmd --no-dangling"
 229                 # no need for --no-progress (v1.7.9+) since stderr will not be a tty
 230                 cmd="$cmd 2>&1"
 231         else
 232                 # use git rev-list --objects --all
 233                 cmd="git rev-list --objects --all${reflog:+ --reflog}${indexedobj:+ --indexed-objects}"
 234                 # but we only want stderr output not any of the objects list
 235                 cmd="$cmd 2>&1 >/dev/null"
 236         fi
 237         [ -z "$hasionice" ] || cmd="ionice -c 3 $cmd"
 238         [ -z "$hasnice" ] || cmd="nice -n 19 $cmd"
 239         fsckresult=0
 240         fsckoutput="$(eval "$cmd")" || fsckresult=$?
 241         if [ -n "$fullcheck" ] && [ -z "$var_have_git_1710" ]; then
 242                 # strip lines starting with "dangling" since --no-dangling is not supported
 243                 # note that "dangling" is NOT translated
 244                 fsckoutput="$(printf '%s\n' "$fsckoutput" | LC_ALL=C sed -n '/^dangling/!p')"
 245         fi
 246         [ -z "$fsckoutput" ] || printf '%s\n' "$fsckoutput"
 247         return $fsckresult
 248 )
 249
 250 cd "$cfg_reporoot"
 251 absroot="$(pwd -P)"
 252 # howmany is how many fsck was run on plus how many were empty
 253 howmany=0
 254 # mtcount is how many were empty
 255 mtcount=0
 256 # okcount is how many fsck returned 0 status for
 257 okcount=0
 258 # warncount is how many fsck returned 0 status but non-empty output
 259 warncount=0
 260 # errresults are results from fsck non-0 status fsck runs
 261 errresults=
 262 # warnresults are non-empty results from 0 status fsck runs
 263 warnresults=
 264 # list of all projects checked in order if --list is active
 265 allprojs=
 266
 267 while IFS='' read -r proj; do
 268         if [ -L "$proj" ]; then
 269                 # symlinks to elsewhere under $cfg_reporoot are ignored
 270                 # symlinks to outside there are reported on unless orphaned
 271                 absproj="$(cd "$proj" && pwd -P)" && [ -n "$absproj" ] &&
 272                         [ -d "$absproj" ] || continue
 273                 case "$absproj" in "$absroot"/*)
 274                         continue
 275                 esac
 276         fi
 277         proj="${proj#./}"
 278         proj="${proj%.git}"
 279         is_listed_proj "$proj" && is_git_dir "$cfg_reporoot/$proj.git" || continue
 280         [ -d "$proj.git/objects" ] || continue
 281         [ "${shownames:-0}" = "0" ] || allprojs="${allprojs:+$allprojs }$proj"
 282         howmany="$(( $howmany + 1 ))"
 283         if is_empty_proj "$proj"; then
 284                 mtcount="$(( $mtcount + 1 ))"
 285                 continue
 286         fi
 287         ok=1
 288         output="$(get_check_proj "$proj")" || ok=
 289         [ -z "$ok" ] || okcount="$(( $okcount + 1 ))"
 290         [ -n "$ok" ] || [ -n "$output" ] || output="git fsck failed with no output"
 291         if [ -n "$output" ]; then
 292                 output="$(printf '%s\n' "$output" | LC_ALL=C sed 's/^/    /')$nl"
 293                 if [ -n "$ok" ]; then
 294                         # warning
 295                         warncount="$(( $warncount + 1 ))"
 296                         [ -z "$warnresults" ] || warnresults="$warnresults$nl"
 297                         warnresults="$warnresults$proj: (warnings only)$nl$output"
 298                 else
 299                         [ -z "$errresults" ] || errresults="$errresults$nl"
 300                         errresults="$errresults$proj: (errors found)$nl$output"
 301                 fi
 302         fi
 303 done <<EOT
 304 $(
 305         if [ -n "$onlylist" ]; then
 306                 printf '%s\n' $onlylist
 307         else
 308                 find -L . -type d \( -path ./_recyclebin -o -path ./_global -o -name '*.git' -print \) -prune 2>/dev/null |
 309                 LC_ALL=C sort -f
 310         fi
 311 )
 312 EOT
 313
 314 enddate="$(date "+$datefmt")"
 315 [ "$minoutput" != "1" ] || [ "$howmany" != "$(( $okcount + $mtcount ))" ] || [ "$warncount" != "0" ] || exit 0
 316 [ "$minoutput" != "2" ] || [ "$howmany" != "$(( $okcount + $mtcount ))" ] || exit 0
 317 domail=cat
 318 [ -z "$mailresult" ] || domail='mailref "fsck@$cfg_gitweburl" -s "[$cfg_name] Project Fsck Status Report" "$cfg_admin"'
 319 {
 320         cat <<EOT
 321 Project Fsck Status Report
 322 ==========================
 323
 324        Start Time: $startdate
 325          End Time: $enddate
 326
 327  Projects Checked: $howmany
 328     Projects Okay: $(( $okcount + $mtcount )) (passed + warned + empty)
 329
 330   Projects Passed: $(( $okcount - $warncount ))
 331   Projects Warned: $warncount
 332    Projects Empty: $mtcount
 333   Projects Failed: $(( $howmany - $mtcount - $okcount ))
 334 EOT
 335         if [ "${shownames:-0}" != "0" ] && [ -n "$allprojs" ]; then
 336                 prefix="\
 337  Projects Checked: "
 338                 for aproj in $allprojs; do
 339                         printf '%s%s\n' "$prefix" "$aproj"
 340                         prefix="\
 341                    "
 342                 done
 343         fi
 344
 345         echo ""
 346
 347         if [ -n "$errresults" ] || [ -n "$warnresults" ]; then
 348                 echo ""
 349                 echo "Fsck Output"
 350                 echo "-----------"
 351                 echo ""
 352                 printf '%s' "$errresults"
 353                 [ -z "$errresults" ] || [ -z "$warnresults" ] || echo ""
 354                 printf '%s' "$warnresults"
 355                 echo ""
 356         fi
 357 } | eval "$domail"