toolbox/reports/project-disk-use.sh

   1 #!/bin/sh
   2
   3 # Report on project disk use.
   4 # Output can be sent as email to admin with -m
   5 # Automatically runs with nice and ionice (if available)
   6
   7 # Usage: project-disk-use [-m] [top-n-only]
   8
   9 # With -m mail the report to $cfg_admin instead of sending it to stdout
  10
  11 # Shows total disk usage in K bytes for $cfg_reporoot
  12 # The top-n-only (all if not given) repos are then listed by
  13 # decreasing order of disk space.
  14
  15 # Note that any *.git directories that are found in $cfg_reporoot that are
  16 # not listed in $cfg_chroot/etc/group ARE included and have a '!' suffix added.
  17
  18 set -e
  19
  20 datefmt='%Y-%m-%d %H:%M:%S %z'
  21 startdate="$(date "+$datefmt")"
  22
  23 . @basedir@/shlib.sh
  24
  25 mailresult=
  26 if [ "$1" = "-m" ]; then
  27         shift
  28         mailresult=1
  29 fi
  30
  31 hasnice=
  32 ! command -v nice >/dev/null || hasnice=1
  33 hasionice=
  34 ! command -v ionice >/dev/null || hasionice=1
  35
  36 nl='
  37 '
  38
  39 fmtcomma() {
  40         # remove leading 0s
  41         _y="${1%%[1-9]*}"; _x="${1#$_y}"; _x="${_x:-0}"
  42         # add commas after each group of 3
  43         _y=
  44         while [ $_x -gt 999 ]; do
  45                 _y="$(printf '%03d' $(($_x - $_x / 1000 * 1000)))${_y:+,$_y}"
  46                 _x="$(($_x/1000))"
  47         done
  48         [ $_x -eq 0 ] || _y="$_x${_y:+,$_y}"
  49         echo "${_y:-0}"
  50 }
  51
  52 fmtpct() {
  53         if [ -z "$1" ] || [ "$1" = "0" ]; then return; fi
  54         if [ -z "$2" ] || [ "$2" = "0" ]; then return; fi
  55         _fmtpct=$(( ( $1 * 100 + $2 / 2 ) / $2 ))
  56         if [ $_fmtpct -le 100 ]; then
  57                 _of=
  58                 [ -z "$4" ] || _of=" of $(fmtcomma $2)"
  59                 echo "${3:- }($_fmtpct%$_of)"
  60         fi
  61 }
  62
  63 get_use_k() {
  64         _targ="$1"
  65         shift
  66         _cmd="du $var_du_follow -k -s"
  67         if [ -n "$var_du_exclude" ]; then
  68                 while [ -n "$1" ]; do
  69                         _cmd="$_cmd $var_du_exclude \"$1\""
  70                         shift
  71                 done
  72         elif [ $# -ne 0 ]; then
  73                 echo "get_use_k: error: no du exclude option available" >&2
  74                 exit 1
  75         fi
  76         _cmd="$_cmd \"$_targ\" 2>/dev/null | cut -f 1"
  77         [ -z "$hasionice" ] || _cmd="ionice -c 3 $_cmd"
  78         [ -z "$hasnice" ] || _cmd="nice -n 19 $_cmd"
  79         eval "$_cmd"
  80 }
  81
  82 is_active() (
  83         cd "$cfg_reporoot/$1.git" || return 1
  84         check_interval lastchange 2592000 # 30 days
  85 )
  86
  87 projlist="$(cut -d : -f 1 <"$cfg_chroot/etc/group")"
  88
  89 is_listed_proj() {
  90   echo "$projlist" | grep -q -e "^$1$"
  91 }
  92
  93 totaluse="$(get_use_k "$cfg_reporoot")"
  94 globaluse="$(get_use_k "$cfg_reporoot/_global")"
  95 totaluse="$(( $totaluse - $globaluse ))"
  96 if [ -n "$var_du_exclude" ]; then
  97         # Attribute all hard-linked stuff in _recyclebin to non-_recyclebin
  98         inuse="$(get_use_k "$cfg_reporoot" "_recyclebin")"
  99         inuse="$(( $inuse - $globaluse ))"
 100         binned="$(( $totaluse - $inuse ))"
 101 else
 102         # No du exclude option, if binned contains hard links into other repos
 103         # its size may appear larger than it actually is and the kpct will be off
 104         binned="$(get_use_k "$cfg_reporoot/_recyclebin")"
 105         inuse="$(( $totaluse - $binned ))"
 106 fi
 107
 108 cd "$cfg_reporoot"
 109 absroot="$(pwd -P)"
 110 total=0
 111 ktotal=0
 112 howmany=0
 113 orphans=0
 114 banged=0
 115 bangsent=0
 116 mirrors=0
 117 forks=0
 118 mactive=0
 119 pactive=0
 120 results=
 121 while IFS='' read -r proj; do
 122         external=
 123         absproj="$(cd "$proj" && pwd -P)" && [ -n "$absproj" ] && [ -d "$absproj" ] || continue
 124         if [ -L "$proj" ]; then
 125                 # symlinks to elsewhere under $cfg_reporoot are ignored
 126                 # symlinks to outside there are reported on unless orphaned
 127                 case "$absproj" in "$absroot"/*)
 128                         continue
 129                 esac
 130         fi
 131         case "$absproj" in "$absroot"/*);;*)
 132                 external=1
 133         esac
 134         proj="${proj#./}"
 135         proj="${proj%.git}"
 136         x=
 137         a=
 138         if ! is_listed_proj "$proj"; then
 139                 [ -z "$external" ] || continue
 140                 x='!'
 141                 orphans="$(( $orphans + 1 ))"
 142         fi
 143         case "$proj" in */*) forks="$(( $forks + 1 ))"; esac
 144         mirror="$(get_mirror_type "$proj.git" 2>/dev/null)" || :
 145         [ -z "$mirror" ] || mirrors="$(( $mirrors + 1 ))"
 146         if is_active "$proj"; then
 147                 a='*'
 148                 if [ -n "$mirror" ]; then
 149                         mactive="$(( $mactive + 1 ))"
 150                 else
 151                         pactive="$(( $pactive + 1 ))"
 152                 fi
 153         fi
 154         : ${mirror:=M}
 155         usek="$(get_use_k "$proj.git")"
 156         repok=
 157         girocco_reposizek=
 158         girocco_bang_count=
 159         girocco_bang_firstfail=
 160         girocco_bang_messagesent=
 161         [ -L "$proj.git/objects" ] || ! [ -d "$proj.git/objects" ] ||
 162         eval "$(git --git-dir="$cfg_reporoot/$proj.git" config --get-regexp \
 163                 '^girocco\.((bang\.(count|firstfail|messagesent))|reposizek)$' |
 164                 LC_ALL=C awk '{gsub(/[.]/,"_",$1); $0 ~ / / || sub(/$/," "); sub(/ /,"=\042"); print $0 "\042"}')" || :
 165         b=
 166         if [ -n "$girocco_bang_count" ] && [ "${girocco_bang_count#*[!0-9]}" = "$girocco_bang_count" ] && [ "$girocco_bang_count" -gt 0 ]; then
 167                 banged=$(( $banged + 1 ))
 168                 girocco_bang_count=$(( 0 + $girocco_bang_count ))
 169                 b="~(${girocco_bang_firstfail:+$girocco_bang_firstfail/}$girocco_bang_count"
 170                 if [ "$girocco_bang_messagesent" = "true" ] || [ "$girocco_bang_messagesent" = "1" ]; then
 171                         b="$b/sent"
 172                         bangsent=$(( $bangsent + 1 ))
 173                 fi
 174                 b="$b)"
 175         fi
 176         repok="$girocco_reposizek"
 177         repokpct=
 178         case "$repok" in
 179                 [0-9]*)
 180                         repok="${repok%%[!0-9]*}"
 181                         repokpct=$(( ( $repok * 100 + $usek / 2 ) / $usek ))
 182                         if [ $repokpct -le 100 ]; then
 183                                 repokpct="$repokpct%"
 184                         else
 185                                 repokpct="100+"
 186                         fi;;
 187                 *)
 188                         repok=0
 189                         repokpct=-;;
 190         esac
 191         if [ -z "$external" ]; then
 192                 ktotal=$(( $ktotal + $repok ))
 193                 total="$(( $total + $usek ))"
 194         fi
 195         howmany="$(( $howmany + 1 ))"
 196         line="$usek $repokpct $mirror $proj$a$x$b$nl"
 197         results="$results$line"
 198 done <<EOT
 199 $(find -L . -type d \( -path ./_recyclebin -o -path ./_global -o -name '*.git' -print \) -prune 2>/dev/null)
 200 EOT
 201
 202 kpct=$(( ( $ktotal * 100 + $inuse / 2 ) / $inuse ))
 203 enddate="$(date "+$datefmt")"
 204 domail=cat
 205 [ -z "$mailresult" ] || domail='mailref "diskuse@$cfg_gitweburl" -s "[$cfg_name] Project Disk Use Report" "$cfg_admin"'
 206 {
 207         cat <<EOT
 208 Project Disk Use Report
 209 =======================
 210
 211        Start Time: $startdate
 212          End Time: $enddate
 213
 214   Repository Root: $cfg_reporoot
 215 Unbinned Disk Use: $(fmtcomma "$inuse") (1024-byte blocks)
 216   reposizek Total: $(fmtcomma "$ktotal") ($kpct%)
 217
 218  Recycle Bin Root: $cfg_reporoot/_recyclebin
 219   Binned Disk Use: $(fmtcomma "$binned") (1024-byte blocks)
 220
 221    Total Disk Use: $(fmtcomma "$totaluse") (1024-byte blocks)
 222
 223  Repository Count: $(fmtcomma "$howmany")
 224             Forks: $(fmtcomma "$forks")$(fmtpct "$forks" "$howmany")
 225           Mirrors: $(fmtcomma "$mirrors")$(fmtpct "$mirrors" "$howmany")
 226           ~Banged: $(fmtcomma "$banged")$(fmtpct "$banged" "$howmany")/($(( $banged - $bangsent )) unsent)
 227          Orphaned: $(fmtcomma "$orphans")
 228  Repository Total: $(fmtcomma "$total") (1024-byte blocks)
 229
 230    *30-Day Active: $(fmtcomma "$(( $pactive + $mactive ))")$(fmtpct "$(( $pactive + $mactive ))" "$howmany")
 231              Push: $(fmtcomma "$pactive")$(fmtpct "$pactive" "$howmany")$(fmtpct "$pactive" "$(( $pactive + $mactive ))" "/" 1)$(fmtpct "$pactive" "$(( $howmany - $mirrors ))" "/" 1)
 232            Mirror: $(fmtcomma "$mactive")$(fmtpct "$mactive" "$howmany")$(fmtpct "$mactive" "$(( $pactive + $mactive ))" "/" 1)$(fmtpct "$mactive" "$mirrors" "/" 1)
 233
 234 $(df -h "$cfg_reporoot")
 235 EOT
 236
 237         if [ $# -lt 1 ] || [ $1 != "0" ]; then
 238                 topn=cat
 239                 message="Individual Repository Use"
 240                 case "${1%%[!0-9]*}" in ?*)
 241                         message="Individual Repository Use (Top $1)"
 242                         topn="head -n ${1%%[!0-9]*}"
 243                 esac
 244                 echo ""
 245                 echo ""
 246                 echo "$message"
 247                 echo "$message" | tr -c '\n' -
 248                 echo ""
 249                 printf '%s' "$results" | sort -k1,1nr -k4,4 |
 250                 while read -r a b c d; do
 251                         printf "%10s %4s  %s  %s\n" "$(fmtcomma "$a")" "$b" "$c" "$d"
 252                 done |
 253                 sed -e 's/ [M-] /   /g' | $topn
 254         fi
 255 } | eval "$domail"