A lot of enhancements

collector.sh

@@ -5,6 +5,8 @@
 # The events are then reported to something.
 #
 
+# TODO: Also treat inbound email - maybe even accept things like proxy-accept and proxy-reject?
+
 # Debugging option
 #set -x
 
@@ -16,12 +18,26 @@ cd `dirname $0`
 
 
 ### COUNTERS ###
-log_lines=0
+log_lines_total=0
-mail_sent=0
+log_lines_smtp=0
-mail_deferred=0
+log_lines_smtpd=0
-mail_bounced_total=0
+
-mail_bounced_spam=0
+mo_sent=0
-mail_reputation=0
+mo_deferred=0
+mo_bounced_total=0
+mo_bounced_spam=0
+mo_bounced_reputation=0
+mo_deliverable=0
+mo_undeliverable=0
+
+mi_accept=0
+mi_reject=0
+mi_proxy_accept=0
+mi_proxy_reject=0
+
+mi_connects=0
+mi_connects_tls=0
+
 
 ### First time setup (if required)
 if [ ! -f "$DBFILE" ]; then
@@ -33,12 +49,22 @@ if [ ! -f "$DBFILE" ]; then
     rrdtool create $RRDFILE \
         --start `date +%s` \
         --step 60 \
-        DS:log_lines:GAUGE:60:0:U \
+        DS:log_lines_total:GAUGE:60:0:U \
-        DS:mail_sent:GAUGE:60:0:U \
+        DS:log_lines_smtp:GAUGE:60:0:U \
-        DS:mail_deferred:GAUGE:60:0:U \
+        DS:log_lines_smtpd:GAUGE:60:0:U \
-        DS:mail_bounced_total:GAUGE:60:0:U \
+        DS:mo_sent:GAUGE:60:0:U \
-        DS:mail_bounced_spam:GAUGE:60:0:U \
+        DS:mo_deferred:GAUGE:60:0:U \
-        DS:mail_reputation:GAUGE:60:0:U \
+        DS:mo_bounced_total:GAUGE:60:0:U \
+        DS:mo_bounced_spam:GAUGE:60:0:U \
+        DS:mo_bounced_reputation:GAUGE:60:0:U \
+        DS:mo_deliverable:GAUGE:60:0:U \
+        DS:mo_undeliverable:GAUGE:60:0:U \
+        DS:mi_accept:GAUGE:60:0:U \
+        DS:mi_reject:GAUGE:60:0:U \
+        DS:mi_proxy_accept:GAUGE:60:0:U \
+        DS:mi_proxy_reject:GAUGE:60:0:U \
+        DS:mi_connects:GAUGE:60:0:U \
+        DS:mi_connects_tls:GAUGE:60:0:U \
         RRA:AVERAGE:0.5:1:1440 \
         RRA:AVERAGE:0.5:15:1344 \
         RRA:AVERAGE:0.5:30:720 \
@@ -72,20 +98,37 @@ if [ $new_log_position -lt $old_log_position ]; then
     old_log_position=0
 fi
 
-# Calculate number of log lines
+# Collect number of log lines
-log_lines=`expr $new_log_position - $old_log_position`
+log_lines_total=`expr $new_log_position - $old_log_position`
+log_lines_smtp=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' wc -l)
+log_lines_smtpd=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtpd\[' wc -l)
 
 # Get lines and analyse them a bit, do the counting
-mail_sent=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep 'status=sent' | grep -v 'relay=127.0.0.1' | wc -l)
+
-mail_deferred=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep 'status=deferred' | wc -l)
+# Collect outbound email statistics - localhost is excluded.
-mail_bounced_total=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep 'status=bounced' | wc -l)
+mo_sent=$(              sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=sent'          | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l)
-mail_bounced_spam=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep 'status=bounced' | grep -i 'spam' | wc -l)
+mo_deferred=$(          sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=deferred'      | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l)
-mail_reputation=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep -i 'poor reputation' | wc -l)
+mo_deliverable=$(       sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=deliverable'   | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l)
+mo_undeliverable=$(     sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=undeliverable' | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l)
+mo_bounced_total=$(     sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=bounced'       | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l)
+mo_bounced_spam=$(      sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep 'status=bounced' | grep -i 'spam' | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l)
+mo_bounced_reputation=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep -i 'reputation'        | grep -v 'relay=127\.0\.0\.1\|relay=localhost\|relay=::1' | wc -l)
+
+# Collect inbound email statistics
+mi_accept=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtpd\[' | grep '\/smtpd\[' | grep ': [0-9A-F]*: client=' | wc -l)
+mi_reject=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtpd\[' | grep ': NOQUEUE: reject:' | wc -l)
+mi_proxy_accept=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep ': proxy-accept: END-OF-MESSAGE: ' | wc -l)
+mi_proxy_reject=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtp\[' | grep ': proxy-reject: END-OF-MESSAGE: ' | wc -l)
+
+# Collect connection data
+mi_connects=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtpd\[' | grep ': connect from' | wc -l)
+mi_connects_tls=$(sed -n "$old_log_position","$new_log_position"p $LOGFILE | grep '\/smtpd\[' | grep 'Anonymous TLS connection established from' | wc -l)
+
 
 # Report counter results
 rrdtool update $RRDFILE \
-    -t log_lines:mail_sent:mail_deferred:mail_bounced_total:mail_bounced_spam:mail_reputation \
+    -t log_lines_total:log_lines_smtp:log_lines_smtpd:mo_sent:mo_deferred:mo_bounced_total:mo_bounced_spam:mo_bounced_reputation:mo_deliverable:mo_undeliverable:mi_accept:mi_reject:mi_proxy_accept:mi_proxy_reject:mi_connects:mi_connects_tls \
-    $db_last_modified:$log_lines:$mail_sent:$mail_deferred:$mail_bounced_total:$mail_bounced_spam:$mail_reputation
+$db_last_modified:$log_lines_total:$log_lines_smtp:$log_lines_smtpd:$mo_sent:$mo_deferred:$mo_bounced_total:$mo_bounced_spam:$mo_bounced_reputation:$mo_deliverable:$mo_undeliverable:$mi_accept:$mi_reject:$mi_proxy_accept:$mi_proxy_reject:$mi_connects:$mi_connects_tls
 
 # That's it.
 exit 0

grapher.sh

@@ -9,7 +9,7 @@ cd `dirname $0`
 rrdtool graph $GRAPHDIR/hour.png \
     --start end-1h \
     --step 60 \
-    --title "Mail statistics for `hostname -f`" \
+    --title "Mail statistics for `hostname -f` - Last hour" \
     --vertical-label "mails/minute" \
     -h 200 -w 800 --slope-mode \
     DEF:log_lines=$RRDFILE:log_lines:AVERAGE \
@@ -17,22 +17,22 @@ rrdtool graph $GRAPHDIR/hour.png \
     DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \
     DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \
     DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \
-    DEF:mail_reputation=$RRDFILE:mail_reputation:AVERAGE \
+    DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \
-    AREA:mail_deferred#77777733: \
+    AREA:mail_deferred#77777733 \
-    LINE:mail_deferred#777777:'deferred': \
+    LINE:mail_deferred#777777:'deferred' \
-    AREA:mail_sent#00DD0055: \
+    AREA:mail_sent#00DD0055 \
-    LINE:mail_sent#00DD00:'sent': \
+    LINE:mail_sent#00DD00:'sent' \
-    AREA:mail_bounced_total#0000FF77: \
+    AREA:mail_bounced_total#0000FF77 \
-    LINE:mail_bounced_total#0000FF:'bounced total': \
+    LINE:mail_bounced_total#0000FF:'bounced total' \
-    AREA:mail_bounced_spam#FF0000AA: \
+    AREA:mail_bounced_spam#FF0000AA \
-    LINE:mail_bounced_spam#FF0000:'bounced due to SPAM': \
+    LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \
-    AREA:mail_reputation#FF00FFFF: \
+    AREA:mail_bounced_reputation#FF00FFFF \
-    LINE:mail_reputation#FF00FFFF:'reputation event':
+    LINE:mail_bounced_reputation#FF00FFFF:'reputation event'
 
 rrdtool graph $GRAPHDIR/6hour.png \
     --start end-6h \
     --step 60 \
-    --title "Mail statistics for `hostname -f`" \
+    --title "Mail statistics for `hostname -f` - Last 6 hours" \
     --vertical-label "mails/minute" \
     -h 200 -w 800 --slope-mode \
     DEF:log_lines=$RRDFILE:log_lines:AVERAGE \
@@ -40,22 +40,22 @@ rrdtool graph $GRAPHDIR/6hour.png \
     DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \
     DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \
     DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \
-    DEF:mail_reputation=$RRDFILE:mail_reputation:AVERAGE \
+    DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \
-    AREA:mail_deferred#77777733: \
+    AREA:mail_deferred#77777733 \
-    LINE:mail_deferred#777777:'deferred': \
+    LINE:mail_deferred#777777:'deferred' \
-    AREA:mail_sent#00DD0055: \
+    AREA:mail_sent#00DD0055 \
-    LINE:mail_sent#00DD00:'sent': \
+    LINE:mail_sent#00DD00:'sent' \
-    AREA:mail_bounced_total#0000FF77: \
+    AREA:mail_bounced_total#0000FF77 \
-    LINE:mail_bounced_total#0000FF:'bounced total': \
+    LINE:mail_bounced_total#0000FF:'bounced total' \
-    AREA:mail_bounced_spam#FF0000AA: \
+    AREA:mail_bounced_spam#FF0000AA \
-    LINE:mail_bounced_spam#FF0000:'bounced due to SPAM': \
+    LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \
-    AREA:mail_reputation#FF00FFFF: \
+    AREA:mail_bounced_reputation#FF00FFFF \
-    LINE:mail_reputation#FF00FFFF:'reputation event':
+    LINE:mail_bounced_reputation#FF00FFFF:'reputation event'
 
 rrdtool graph $GRAPHDIR/day.png \
     --start end-24h \
     --step 60 \
-    --title "Mail statistics for `hostname -f`" \
+    --title "Mail statistics for `hostname -f` - Last day" \
     --vertical-label "mails/minute" \
     -h 200 -w 800 --slope-mode \
     DEF:log_lines=$RRDFILE:log_lines:AVERAGE \
@@ -63,22 +63,22 @@ rrdtool graph $GRAPHDIR/day.png \
     DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \
     DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \
     DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \
-    DEF:mail_reputation=$RRDFILE:mail_reputation:AVERAGE \
+    DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \
-    AREA:mail_deferred#77777733: \
+    AREA:mail_deferred#77777733 \
-    LINE:mail_deferred#777777:'deferred': \
+    LINE:mail_deferred#777777:'deferred' \
-    AREA:mail_sent#00DD0055: \
+    AREA:mail_sent#00DD0055 \
-    LINE:mail_sent#00DD00:'sent': \
+    LINE:mail_sent#00DD00:'sent' \
-    AREA:mail_bounced_total#0000FF77: \
+    AREA:mail_bounced_total#0000FF77 \
-    LINE:mail_bounced_total#0000FF:'bounced total': \
+    LINE:mail_bounced_total#0000FF:'bounced total' \
-    AREA:mail_bounced_spam#FF0000AA: \
+    AREA:mail_bounced_spam#FF0000AA \
-    LINE:mail_bounced_spam#FF0000:'bounced due to SPAM': \
+    LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \
-    AREA:mail_reputation#FF00FFFF: \
+    AREA:mail_bounced_reputation#FF00FFFF \
-    LINE:mail_reputation#FF00FFFF:'reputation event':
+    LINE:mail_bounced_reputation#FF00FFFF:'reputation event'
 
 rrdtool graph $GRAPHDIR/week.png \
     --start end-7d \
     --step 60 \
-    --title "Mail statistics for `hostname -f`" \
+    --title "Mail statistics for `hostname -f` - Last week" \
     --vertical-label "mails/minute" \
     -h 200 -w 800 --slope-mode \
     DEF:log_lines=$RRDFILE:log_lines:AVERAGE \
@@ -86,15 +86,61 @@ rrdtool graph $GRAPHDIR/week.png \
     DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \
     DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \
     DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \
-    DEF:mail_reputation=$RRDFILE:mail_reputation:AVERAGE \
+    DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \
-    AREA:mail_deferred#77777733: \
+    AREA:mail_deferred#77777733 \
-    LINE:mail_deferred#777777:'deferred': \
+    LINE:mail_deferred#777777:'deferred' \
-    AREA:mail_sent#00DD0055: \
+    AREA:mail_sent#00DD0055 \
-    LINE:mail_sent#00DD00:'sent': \
+    LINE:mail_sent#00DD00:'sent' \
-    AREA:mail_bounced_total#0000FF77: \
+    AREA:mail_bounced_total#0000FF77 \
-    LINE:mail_bounced_total#0000FF:'bounced total': \
+    LINE:mail_bounced_total#0000FF:'bounced total' \
-    AREA:mail_bounced_spam#FF0000AA: \
+    AREA:mail_bounced_spam#FF0000AA \
-    LINE:mail_bounced_spam#FF0000:'bounced due to SPAM': \
+    LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \
-    AREA:mail_reputation#FF00FFFF: \
+    AREA:mail_bounced_reputation#FF00FFFF \
-    LINE:mail_reputation#FF00FFFF:'reputation event':
+    LINE:mail_bounced_reputation#FF00FFFF:'reputation event'
+
+rrdtool graph $GRAPHDIR/month.png \
+    --start end-1m \
+    --step 60 \
+    --title "Mail statistics for `hostname -f` - Last month" \
+    --vertical-label "mails/minute" \
+    -h 200 -w 800 --slope-mode \
+    DEF:log_lines=$RRDFILE:log_lines:AVERAGE \
+    DEF:mail_sent=$RRDFILE:mail_sent:AVERAGE \
+    DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \
+    DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \
+    DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \
+    DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \
+    AREA:mail_deferred#77777733 \
+    LINE:mail_deferred#777777:'deferred' \
+    AREA:mail_sent#00DD0055 \
+    LINE:mail_sent#00DD00:'sent' \
+    AREA:mail_bounced_total#0000FF77 \
+    LINE:mail_bounced_total#0000FF:'bounced total' \
+    AREA:mail_bounced_spam#FF0000AA \
+    LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \
+    AREA:mail_bounced_reputation#FF00FFFF \
+    LINE:mail_bounced_reputation#FF00FFFF:'reputation event'
+
+rrdtool graph $GRAPHDIR/year.png \
+    --start end-1y \
+    --step 60 \
+    --title "Mail statistics for `hostname -f` - Last year" \
+    --vertical-label "mails/minute" \
+    -h 200 -w 800 --slope-mode \
+    DEF:log_lines=$RRDFILE:log_lines:AVERAGE \
+    DEF:mail_sent=$RRDFILE:mail_sent:AVERAGE \
+    DEF:mail_deferred=$RRDFILE:mail_deferred:AVERAGE \
+    DEF:mail_bounced_total=$RRDFILE:mail_bounced_total:AVERAGE \
+    DEF:mail_bounced_spam=$RRDFILE:mail_bounced_spam:AVERAGE \
+    DEF:mail_bounced_reputation=$RRDFILE:mail_bounced_reputation:AVERAGE \
+    AREA:mail_deferred#77777733 \
+    LINE:mail_deferred#777777:'deferred' \
+    AREA:mail_sent#00DD0055 \
+    LINE:mail_sent#00DD00:'sent' \
+    AREA:mail_bounced_total#0000FF77 \
+    LINE:mail_bounced_total#0000FF:'bounced total' \
+    AREA:mail_bounced_spam#FF0000AA \
+    LINE:mail_bounced_spam#FF0000:'bounced due to SPAM' \
+    AREA:mail_bounced_reputation#FF00FFFF \
+    LINE:mail_bounced_reputation#FF00FFFF:'reputation event'
 

settings.sh

@@ -4,4 +4,4 @@
 LOGFILE="/var/log/mail.info"
 DBFILE="/root/mailstats.db"
 RRDFILE="/root/mailstats.rrd"
-GRAPHDIR="/var/www/html/stats"
+GRAPHDIR="/var/www/jpt.lu/mailstats"

templates/index.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Postfix statistics</title>
+  </head>
+  <body>
+    <img src="hour.png">
+    <img src="6hour.png">
+    <img src="day.png">
+    <img src="week.png">
+    <img src="month.png">
+    <img src="year.png">
+  </body>
+</html>

templates/postfixstats.cron

@@ -0,0 +1,2 @@
+# This cronjob triggers the grapher component of postfix-stats at regular intervals
+*/5 *	* * *	root	/home/jpt/postfix-stats/grapher.sh >/dev/null 2>&1

templates/postfixstats.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Postfix statistics collector service
+After=network.target
+
+[Service]
+Type=simple
+User=root
+Group=root
+WorkingDirectory=/home/jpt/postfix-stats
+ExecStart=/home/jpt/postfix-stats/collector_runner.sh
+
+[Install]
+WantedBy=multi-user.target
+