From 11fd5e0a715098ed16e050a668f514c3045304ce Mon Sep 17 00:00:00 2001 From: Jan Philipp Timme Date: Fri, 4 May 2018 15:20:04 +0200 Subject: [PATCH] A lot of enhancements --- collector.sh | 85 +++++++++++++++----- grapher.sh | 142 ++++++++++++++++++++++----------- settings.sh | 2 +- templates/index.html | 14 ++++ templates/postfixstats.cron | 2 + templates/postfixstats.service | 14 ++++ 6 files changed, 189 insertions(+), 70 deletions(-) create mode 100644 templates/index.html create mode 100644 templates/postfixstats.cron create mode 100644 templates/postfixstats.service diff --git a/collector.sh b/collector.sh index 6e076ed..cc64db2 100755 --- a/collector.sh +++ b/collector.sh @@ -5,6 +5,8 @@ # The events are then reported to something. # +# TODO: Also treat inbound email - maybe even accept things like proxy-accept and proxy-reject? + # Debugging option #set -x @@ -16,12 +18,26 @@ cd `dirname $0` ### COUNTERS ### -log_lines=0 -mail_sent=0 -mail_deferred=0 -mail_bounced_total=0 -mail_bounced_spam=0 -mail_reputation=0 +log_lines_total=0 +log_lines_smtp=0 +log_lines_smtpd=0 + +mo_sent=0 +mo_deferred=0 +mo_bounced_total=0 +mo_bounced_spam=0 +mo_bounced_reputation=0 +mo_deliverable=0 +mo_undeliverable=0 + +mi_accept=0 +mi_reject=0 +mi_proxy_accept=0 +mi_proxy_reject=0 + +mi_connects=0 +mi_connects_tls=0 + ### First time setup (if required) if [ ! -f "$DBFILE" ]; then @@ -33,12 +49,22 @@ if [ ! -f "$DBFILE" ]; then rrdtool create $RRDFILE \ --start `date +%s` \ --step 60 \ - DS:log_lines:GAUGE:60:0:U \ - DS:mail_sent:GAUGE:60:0:U \ - DS:mail_deferred:GAUGE:60:0:U \ - DS:mail_bounced_total:GAUGE:60:0:U \ - DS:mail_bounced_spam:GAUGE:60:0:U \ - DS:mail_reputation:GAUGE:60:0:U \ + DS:log_lines_total:GAUGE:60:0:U \ + DS:log_lines_smtp:GAUGE:60:0:U \ + DS:log_lines_smtpd:GAUGE:60:0:U \ + DS:mo_sent:GAUGE:60:0:U \ + DS:mo_deferred:GAUGE:60:0:U \ + DS:mo_bounced_total:GAUGE:60:0:U \ + DS:mo_bounced_spam:GAUGE:60:0:U \ + DS:mo_bounced_reputation:GAUGE:60:0:U \ + DS:mo_deliverable:GAUGE:60:0:U \ + DS:mo_undeliverable:GAUGE:60:0:U \ + DS:mi_accept:GAUGE:60:0:U \ + DS:mi_reject:GAUGE:60:0:U \ + DS:mi_proxy_accept:GAUGE:60:0:U \ + DS:mi_proxy_reject:GAUGE:60:0:U \ + DS:mi_connects:GAUGE:60:0:U \ + DS:mi_connects_tls:GAUGE:60:0:U \ RRA:AVERAGE:0.5:1:1440 \ RRA:AVERAGE:0.5:15:1344 \ RRA:AVERAGE:0.5:30:720 \ @@ -72,20 +98,37 @@ if [ $new_log_position -lt $old_log_position ]; then old_log_position=0 fi -# Calculate number of log lines -log_lines=`expr $new_log_position - $old_log_position` +# Collect number of log lines +log_lines_total=`expr $new_log_position - $old_log_position` +log_lines_smtp=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' wc -l) +log_lines_smtpd=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtpd\[' wc -l) # Get lines and analyse them a bit, do the counting -mail_sent=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep 'status=sent' | grep -v 'relay=127.0.0.1' | wc -l) -mail_deferred=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep 'status=deferred' | wc -l) -mail_bounced_total=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep 'status=bounced' | wc -l) -mail_bounced_spam=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep 'status=bounced' | grep -i 'spam' | wc -l) -mail_reputation=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep -i 'poor reputation' | wc -l) + +# Collect outbound email statistics - localhost is excluded. +mo_sent=$( sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=sent' | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l) +mo_deferred=$( sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=deferred' | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l) +mo_deliverable=$( sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=deliverable' | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l) +mo_undeliverable=$( sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=undeliverable' | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l) +mo_bounced_total=$( sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=bounced' | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l) +mo_bounced_spam=$( sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=bounced' | grep -i 'spam' | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l) +mo_bounced_reputation=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep -i 'reputation' | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l) + +# Collect inbound email statistics +mi_accept=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtpd\[' | grep '\/smtpd\[' | grep ': [0-9A-F]*: client=' | wc -l) +mi_reject=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtpd\[' | grep ': NOQUEUE: reject:' | wc -l) +mi_proxy_accept=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep ': proxy-accept: END-OF-MESSAGE: ' | wc -l) +mi_proxy_reject=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep ': proxy-reject: END-OF-MESSAGE: ' | wc -l) + +# Collect connection data +mi_connects=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtpd\[' | grep ': connect from' | wc -l) +mi_connects_tls=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtpd\[' | grep 'Anonymous TLS connection established from' | wc -l) + # Report counter results rrdtool update $RRDFILE \ - -t log_lines:mail_sent:mail_deferred:mail_bounced_total:mail_bounced_spam:mail_reputation \ - $db_last_modified:$log_lines:$mail_sent:$mail_deferred:$mail_bounced_total:$mail_bounced_spam:$mail_reputation + -t log_lines_total:log_lines_smtp:log_lines_smtpd:mo_sent:mo_deferred:mo_bounced_total:mo_bounced_spam:mo_bounced_reputation:mo_deliverable:mo_undeliverable:mi_accept:mi_reject:mi_proxy_accept:mi_proxy_reject:mi_connects:mi_connects_tls \ +$db_last_modified:$log_lines_total:$log_lines_smtp:$log_lines_smtpd:$mo_sent:$mo_deferred:$mo_bounced_total:$mo_bounced_spam:$mo_bounced_reputation:$mo_deliverable:$mo_undeliverable:$mi_accept:$mi_reject:$mi_proxy_accept:$mi_proxy_reject:$mi_connects:$mi_connects_tls # That's it. exit 0 diff --git a/grapher.sh b/grapher.sh index b61ba8f..6eb65d0 100755 --- a/grapher.sh +++ b/grapher.sh @@ -9,7 +9,7 @@ cd `dirname $0` rrdtool graph $GRAPHDIR/hour.png \ --start end-1h \ --step 60 \ - --title "Mail statistics for `hostname -f`" \ + --title "Mail statistics for `hostname -f` - Last hour" \ --vertical-label "mails/minute" \ -h 200 -w 800 --slope-mode \ DEF:log_lines=$RRDFILE:log_lines:AVERAGE \ @@ -17,22 +17,22 @@ rrdtool graph $GRAPHDIR/hour.png \ DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \ DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \ DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \ - DEF:mail_reputation=$RRDFILE:mail_reputation:AVERAGE \ - AREA:mail_deferred#77777733: \ - LINE:mail_deferred#777777:'deferred': \ - AREA:mail_sent#00DD0055: \ - LINE:mail_sent#00DD00:'sent': \ - AREA:mail_bounced_total#0000FF77: \ - LINE:mail_bounced_total#0000FF:'bounced total': \ - AREA:mail_bounced_spam#FF0000AA: \ - LINE:mail_bounced_spam#FF0000:'bounced due to SPAM': \ - AREA:mail_reputation#FF00FFFF: \ - LINE:mail_reputation#FF00FFFF:'reputation event': + DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \ + AREA:mail_deferred#77777733 \ + LINE:mail_deferred#777777:'deferred' \ + AREA:mail_sent#00DD0055 \ + LINE:mail_sent#00DD00:'sent' \ + AREA:mail_bounced_total#0000FF77 \ + LINE:mail_bounced_total#0000FF:'bounced total' \ + AREA:mail_bounced_spam#FF0000AA \ + LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \ + AREA:mail_bounced_reputation#FF00FFFF \ + LINE:mail_bounced_reputation#FF00FFFF:'reputation event' rrdtool graph $GRAPHDIR/6hour.png \ --start end-6h \ --step 60 \ - --title "Mail statistics for `hostname -f`" \ + --title "Mail statistics for `hostname -f` - Last 6 hours" \ --vertical-label "mails/minute" \ -h 200 -w 800 --slope-mode \ DEF:log_lines=$RRDFILE:log_lines:AVERAGE \ @@ -40,22 +40,22 @@ rrdtool graph $GRAPHDIR/6hour.png \ DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \ DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \ DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \ - DEF:mail_reputation=$RRDFILE:mail_reputation:AVERAGE \ - AREA:mail_deferred#77777733: \ - LINE:mail_deferred#777777:'deferred': \ - AREA:mail_sent#00DD0055: \ - LINE:mail_sent#00DD00:'sent': \ - AREA:mail_bounced_total#0000FF77: \ - LINE:mail_bounced_total#0000FF:'bounced total': \ - AREA:mail_bounced_spam#FF0000AA: \ - LINE:mail_bounced_spam#FF0000:'bounced due to SPAM': \ - AREA:mail_reputation#FF00FFFF: \ - LINE:mail_reputation#FF00FFFF:'reputation event': + DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \ + AREA:mail_deferred#77777733 \ + LINE:mail_deferred#777777:'deferred' \ + AREA:mail_sent#00DD0055 \ + LINE:mail_sent#00DD00:'sent' \ + AREA:mail_bounced_total#0000FF77 \ + LINE:mail_bounced_total#0000FF:'bounced total' \ + AREA:mail_bounced_spam#FF0000AA \ + LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \ + AREA:mail_bounced_reputation#FF00FFFF \ + LINE:mail_bounced_reputation#FF00FFFF:'reputation event' rrdtool graph $GRAPHDIR/day.png \ --start end-24h \ --step 60 \ - --title "Mail statistics for `hostname -f`" \ + --title "Mail statistics for `hostname -f` - Last day" \ --vertical-label "mails/minute" \ -h 200 -w 800 --slope-mode \ DEF:log_lines=$RRDFILE:log_lines:AVERAGE \ @@ -63,22 +63,22 @@ rrdtool graph $GRAPHDIR/day.png \ DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \ DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \ DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \ - DEF:mail_reputation=$RRDFILE:mail_reputation:AVERAGE \ - AREA:mail_deferred#77777733: \ - LINE:mail_deferred#777777:'deferred': \ - AREA:mail_sent#00DD0055: \ - LINE:mail_sent#00DD00:'sent': \ - AREA:mail_bounced_total#0000FF77: \ - LINE:mail_bounced_total#0000FF:'bounced total': \ - AREA:mail_bounced_spam#FF0000AA: \ - LINE:mail_bounced_spam#FF0000:'bounced due to SPAM': \ - AREA:mail_reputation#FF00FFFF: \ - LINE:mail_reputation#FF00FFFF:'reputation event': + DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \ + AREA:mail_deferred#77777733 \ + LINE:mail_deferred#777777:'deferred' \ + AREA:mail_sent#00DD0055 \ + LINE:mail_sent#00DD00:'sent' \ + AREA:mail_bounced_total#0000FF77 \ + LINE:mail_bounced_total#0000FF:'bounced total' \ + AREA:mail_bounced_spam#FF0000AA \ + LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \ + AREA:mail_bounced_reputation#FF00FFFF \ + LINE:mail_bounced_reputation#FF00FFFF:'reputation event' rrdtool graph $GRAPHDIR/week.png \ --start end-7d \ --step 60 \ - --title "Mail statistics for `hostname -f`" \ + --title "Mail statistics for `hostname -f` - Last week" \ --vertical-label "mails/minute" \ -h 200 -w 800 --slope-mode \ DEF:log_lines=$RRDFILE:log_lines:AVERAGE \ @@ -86,15 +86,61 @@ rrdtool graph $GRAPHDIR/week.png \ DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \ DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \ DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \ - DEF:mail_reputation=$RRDFILE:mail_reputation:AVERAGE \ - AREA:mail_deferred#77777733: \ - LINE:mail_deferred#777777:'deferred': \ - AREA:mail_sent#00DD0055: \ - LINE:mail_sent#00DD00:'sent': \ - AREA:mail_bounced_total#0000FF77: \ - LINE:mail_bounced_total#0000FF:'bounced total': \ - AREA:mail_bounced_spam#FF0000AA: \ - LINE:mail_bounced_spam#FF0000:'bounced due to SPAM': \ - AREA:mail_reputation#FF00FFFF: \ - LINE:mail_reputation#FF00FFFF:'reputation event': + DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \ + AREA:mail_deferred#77777733 \ + LINE:mail_deferred#777777:'deferred' \ + AREA:mail_sent#00DD0055 \ + LINE:mail_sent#00DD00:'sent' \ + AREA:mail_bounced_total#0000FF77 \ + LINE:mail_bounced_total#0000FF:'bounced total' \ + AREA:mail_bounced_spam#FF0000AA \ + LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \ + AREA:mail_bounced_reputation#FF00FFFF \ + LINE:mail_bounced_reputation#FF00FFFF:'reputation event' + +rrdtool graph $GRAPHDIR/month.png \ + --start end-1m \ + --step 60 \ + --title "Mail statistics for `hostname -f` - Last month" \ + --vertical-label "mails/minute" \ + -h 200 -w 800 --slope-mode \ + DEF:log_lines=$RRDFILE:log_lines:AVERAGE \ + DEF:mail_sent=$RRDFILE:mail_sent:AVERAGE \ + DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \ + DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \ + DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \ + DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \ + AREA:mail_deferred#77777733 \ + LINE:mail_deferred#777777:'deferred' \ + AREA:mail_sent#00DD0055 \ + LINE:mail_sent#00DD00:'sent' \ + AREA:mail_bounced_total#0000FF77 \ + LINE:mail_bounced_total#0000FF:'bounced total' \ + AREA:mail_bounced_spam#FF0000AA \ + LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \ + AREA:mail_bounced_reputation#FF00FFFF \ + LINE:mail_bounced_reputation#FF00FFFF:'reputation event' + +rrdtool graph $GRAPHDIR/year.png \ + --start end-1y \ + --step 60 \ + --title "Mail statistics for `hostname -f` - Last year" \ + --vertical-label "mails/minute" \ + -h 200 -w 800 --slope-mode \ + DEF:log_lines=$RRDFILE:log_lines:AVERAGE \ + DEF:mail_sent=$RRDFILE:mail_sent:AVERAGE \ + DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \ + DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \ + DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \ + DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \ + AREA:mail_deferred#77777733 \ + LINE:mail_deferred#777777:'deferred' \ + AREA:mail_sent#00DD0055 \ + LINE:mail_sent#00DD00:'sent' \ + AREA:mail_bounced_total#0000FF77 \ + LINE:mail_bounced_total#0000FF:'bounced total' \ + AREA:mail_bounced_spam#FF0000AA \ + LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \ + AREA:mail_bounced_reputation#FF00FFFF \ + LINE:mail_bounced_reputation#FF00FFFF:'reputation event' diff --git a/settings.sh b/settings.sh index d94b5a2..6a01581 100755 --- a/settings.sh +++ b/settings.sh @@ -4,4 +4,4 @@ LOGFILE="/var/log/mail.info" DBFILE="/root/mailstats.db" RRDFILE="/root/mailstats.rrd" -GRAPHDIR="/var/www/html/stats" +GRAPHDIR="/var/www/jpt.lu/mailstats" diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..f3dd097 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,14 @@ + + + + Postfix statistics + + + + + + + + + + diff --git a/templates/postfixstats.cron b/templates/postfixstats.cron new file mode 100644 index 0000000..38a6cf5 --- /dev/null +++ b/templates/postfixstats.cron @@ -0,0 +1,2 @@ +# This cronjob triggers the grapher component of postfix-stats at regular intervals +*/5 * * * * root /home/jpt/postfix-stats/grapher.sh >/dev/null 2>&1 diff --git a/templates/postfixstats.service b/templates/postfixstats.service new file mode 100644 index 0000000..7a3408d --- /dev/null +++ b/templates/postfixstats.service @@ -0,0 +1,14 @@ +[Unit] +Description=Postfix statistics collector service +After=network.target + +[Service] +Type=simple +User=root +Group=root +WorkingDirectory=/home/jpt/postfix-stats +ExecStart=/home/jpt/postfix-stats/collector_runner.sh + +[Install] +WantedBy=multi-user.target +