sendmailanalyzer/sendmailanalyzer

3192 lines
103 KiB
Perl
Executable file
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/perl
#
# SendmailAnalyzer: maillog parser and statistics reports tool for Sendmail
# Copyright (C) 2002-2018 Gilles Darold
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
use vars qw($VERSION $AUTHOR $COPYRIGHT $PROGRAM @ARGS);
use strict;
use Getopt::Long qw(:config no_ignore_case bundling);
use POSIX qw(:sys_wait_h :errno_h :fcntl_h :signal_h);
use MIME::Base64;
use MIME::QuotedPrint;
use IO::File;
$VERSION = '9.2';
$AUTHOR = "Gilles Darold <gilles\@darold.net>";
$COPYRIGHT = "(c) 2002-2018 - Gilles Darold <gilles\@darold.net>";
$SIG{'CHLD'} = 'DEFAULT';
# Keep command line arguments in case we received SIGHUP
$PROGRAM = $0;
push(@ARGS, @ARGV);
# Month translation
my %MONTH_TO_NUM = (
'Jan' => '01',
'Feb' => '02',
'Mar' => '03',
'Apr' => '04',
'May' => '05',
'Jun' => '06',
'Jul' => '07',
'Aug' => '08',
'Sep' => '09',
'Oct' => '10',
'Nov' => '11',
'Dec' => '12'
);
# Configuration storage hash
my %CONFIG = ();
# Other configuration directives
my $CONFIG_FILE = "/usr/local/sendmailanalyzer/sendmailanalyzer.conf";
my $SHOW_VER = 0;
my $INTERACTIVE = 0;
my $HELP = 0;
my $LAST_PARSE_FILE = 'LAST_PARSED';
my $PID_FILE = 'sendmailanalyzer.pid';
my $SA_PIPE = new IO::File;
my $EXIM_REGEX = qr/^(\d+\-\d+\-\d+) (\d+:\d+:\d+) ([A-Za-z0-9]{6}\-[A-Za-z0-9]{6}\-[A-Za-z0-9]{2}) ([\-\*=><]+) /;
my $HOSTNAME = '';
# Global variable to store temporary parsed data
my %SYSERR = ();
my %DSN = ();
my %FROM = ();
my %TO = ();
my %REJECT = ();
my %SPAM = ();
my %VIRUS = ();
my $LAST_PARSED = '';
my %ERRMSG = ();
my %OTHER = ();
my %SPAMDETAIL = ();
my %AUTH = ();
my %GREYLIST = ();
my $KID = '';
my %POSTGREY = ();
my %MSGID = ();
my %SKIPMSG = ();
my %POSTFIX_PLUGIN_TEMP_DELIVERY = ();
my %AMAVIS_ID = ();
my %STARTTLS = ();
my %SPAMPD = ();
my %KEEP_TEMPORARY = ();
my %SPF_DKIM = ();
# Collect command line arguments
GetOptions (
'a|args=s' => \$CONFIG{TAIL_ARGS},
'b|break!' => \$CONFIG{BREAK},
'c|config=s' => \$CONFIG_FILE,
'd|debug!' => \$CONFIG{DEBUG},
'f|full!' => \$CONFIG{FULL},
'F|force!' => \$CONFIG{FORCE},
'g|postgrey=s' => \$CONFIG{POSTGREY_NAME},
'h|help!' => \$HELP,
'i|interactive!' => \$INTERACTIVE,
'j|journalctl=s' => \$CONFIG{JOURNALCTL_CMD},
'l|log=s' => \$CONFIG{LOG_FILE},
'm|mailscanner=s' => \$CONFIG{MAILSCAN_NAME},
'n|clamd=s' => \$CONFIG{CLAMD_NAME},
'o|output=s' => \$CONFIG{OUT_DIR},
'p|piddir=s' => \$CONFIG{PID_DIR},
's|sendmail=s' => \$CONFIG{MTA_NAME},
't|tail=s' => \$CONFIG{TAIL_PROG},
'v|version!' => \$SHOW_VER,
'w|write-delay=i' => \$CONFIG{DELAY},
'z|zcat=s' => \$CONFIG{ZCAT_PROG},
'y|year=s' => \$CONFIG{DEFAULT_YEAR},
'spamd=s' => \$CONFIG{SPAMD_NAME},
'spf=s' => \$CONFIG{SPF_DKIM_NAME},
'hostname=s' => \$HOSTNAME,
);
$CONFIG{FULL} = 1 if ($CONFIG{FORCE});
&usage if ($HELP);
# Read configuration file
&read_config($CONFIG_FILE);
# Checked forced year syntax
if ($CONFIG{DEFAULT_YEAR} && ($CONFIG{DEFAULT_YEAR} !~ /^\d{4}$/)) {
die "FATAL: Default year $CONFIG{DEFAULT_YEAR} should be 4 digits!\n";
}
# Check if output dir exist and we can write file
if (!-d $CONFIG{OUT_DIR}) {
die "FATAL: Output directory $CONFIG{OUT_DIR} should exists !\n";
} else {
open(OUT, ">$CONFIG{OUT_DIR}/test.dat") or die "FATAL: Output directory $CONFIG{OUT_DIR} should be writable !\n";
close(OUT);
unlink("$CONFIG{OUT_DIR}/test.dat");
}
if ($CONFIG{DEBUG}) {
print STDERR "Running in verbose mode...\n";
}
if ($SHOW_VER || $CONFIG{DEBUG}) {
print STDERR "\n\tsendmailanalyzer v$VERSION. $COPYRIGHT\n\n";
exit 0 if ($SHOW_VER);
}
####
# Install signal handlers
####
# Die cleanly on signal
sub terminate
{
close($SA_PIPE) if ($SA_PIPE >= 0);
&flush_data(1);
&dprint("Received terminating signal.", 1);
}
# Restart on signal
sub restart_sa
{
close($SA_PIPE) if ($SA_PIPE >= 0);
&dprint("Received SIGHUP signal: reloading configuration file and reopening log file.");
&flush_data(1);
&clean_globals();
exec($^X, $PROGRAM, @ARGS) or die "FATAL: Couldn't restart: $!\n";
}
# Reload configuration and reopen log file on kill -1
my $sigset_hup = POSIX::SigSet->new();
my $action_hup = POSIX::SigAction->new('restart_sa', $sigset_hup, &POSIX::SA_NODEFER);
POSIX::sigaction(&POSIX::SIGHUP, $action_hup);
# Terminate on kill -15
my $sigset_term = POSIX::SigSet->new();
my $action_term = POSIX::SigAction->new('terminate', $sigset_term, &POSIX::SA_NODEFER);
POSIX::sigaction(&POSIX::SIGTERM, $action_term);
# Terminate on kill -9
my $sigset_int = POSIX::SigSet->new();
my $action_int = POSIX::SigAction->new('terminate', $sigset_int, &POSIX::SA_NODEFER);
POSIX::sigaction(&POSIX::SIGKILL, $action_int);
my $CURRENT_TIME = &format_time(localtime(time));
# Run in interactive mode if required
if ($INTERACTIVE) {
# Start in interactive mode
print "\n*** sendmailanalyzer v$VERSION (pid:$$) started at " . localtime(time) . "\n";
} else {
# detach from terminal
my $pid = fork;
exit 0 if ($pid);
die "Couldn't fork: $!" unless defined($pid);
POSIX::setsid() or die "Can't detach: \$!";
&dprint("Detach from terminal with pid: $$");
}
# Set name of the program without path*
my $orig_name = $0;
$0 = 'sendmailanalyzer';
# Continuously read the maillog file using a pipe to tail program
&dprint("Entering main loop...");
&start_loop;
exit 0;
#-------------------------------- ROUTINES ------------------------------------
####
# Dump usage to STDERR
####
sub usage
{
print STDERR qq{
sendmailanalyzer v$VERSION usage:
-a | --args "tail_args": tail command line arguments. Default "-n 0 -f".
-b | --break : do not run tail after parsing full maillog and exit.
-c | --config file : path to configuration file. Default is to read it from
/etc/sendmailanalyzer.conf.
-d | --debug : turn on debug mode.
-f | --full : parse full maillog and compute stat. Default is to read
LAST_PARSED file to start from last collected event.
-F | --force : same as --full but don't take care of LAST_PARSED file,
that means that log file always contains new entries.
-h | --help : show this short help and exit.
-i | --interactive : run in interactive mode useful if you want day to day
: report. Default is daemon mode, real time statistics.
-j | --journalctl cmd : set the journalctl command to use to replace logfile.
-l | --log file : path to maillog file. Default is /var/log/maillog.
-m | --mailscanner name: syslog MailScanner program name. Default: Mailscanner.
-o | --output dir : path to the output directory where data file will be
written. Default /var/www/htdocs.
-p | --piddir dir : path where pid file will be stored. Default /var/run/.
-s | --sendmail name : syslog sendmail program name. Default sm-mta|sendmail.
-t | --tail tail_prog : path to the tail system command. Default /usr/bin/tail.
-v | --version : show version and exit
-w | --write-delay sec : memory storage delay in second before saving data
to disk. Default: 5 seconds.
-y | --year 2001 : force the years date part of the log to given value.
Default is current year or previous year if log lines
appear in the future.
-z | --zcat zcat_prog : path to the zcat command for compressed maillog.
Default /usr/bin/zcat.
--spamd name : syslog Spamd program name. Default: spamd.
--hostname name : set a hostname for exim logs. Default: unknown.
--spf name : syslog SPF and DKIM program name list.
Default: opendmarc|opendkim.
};
exit 0;
}
####
# Function used to dump debugging information
####
sub dprint
{
my $msg = shift;
my $exit = shift;
print STDERR "DEBUG: $msg\n" if ($exit || $CONFIG{DEBUG});
if ($exit) {
unlink("$CONFIG{PID_DIR}/$PID_FILE");
exit 1;
}
}
####
# Start reading maillog file
####
my $OLD_LAST_PARSED = '';
my $OLD_OFFSET = 0;
sub start_loop
{
if ($CONFIG{FULL}) {
if (!$CONFIG{FORCE}) {
if (-e "$CONFIG{OUT_DIR}/$LAST_PARSE_FILE") {
if ( not open(IN, "$CONFIG{OUT_DIR}/$LAST_PARSE_FILE")) {
&logerror("Can't read file $CONFIG{OUT_DIR}/$LAST_PARSE_FILE: $!");
} else {
my $tmp = <IN>;
chomp($tmp);
($OLD_LAST_PARSED,$OLD_OFFSET) = split(/[\t]/, $tmp);
close(IN);
}
}
}
if ($CONFIG{JOURNALCTL_CMD}) {
$OLD_OFFSET = 0;
my $since = '';
if ( ($CONFIG{JOURNALCTL_CMD} !~ /--since|-S/) && ($OLD_LAST_PARSED =~ /^(\d+)-(\d+)-(\d+).(\d+):(\d+):(\d+)/) ) {
$since = " --since=\"$1-$2-$3 $4:$5:$6\"";
}
&dprint("Parsing full entries from command: $CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\"");
if (!($KID = open(SA_FILE, "$CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\" |"))) {
&dprint("$0: cannot read input from command: $CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\", $!", 1);
}
} else {
&dprint("Parsing full $CONFIG{LOG_FILE}");
if ($CONFIG{LOG_FILE} !~ /\.gz/) {
if (!($KID = open(SA_FILE, "$CONFIG{LOG_FILE}"))) {
&dprint("$0: cannot read $CONFIG{LOG_FILE}: $!", 1);
}
} else {
# Open a pipe to zcat program for compressed log
if (!($KID = open(SA_FILE, "$CONFIG{ZCAT_PROG} $CONFIG{LOG_FILE}|"))) {
&dprint("$0: cannot read from pipe to \"$CONFIG{ZCAT_PROG} $CONFIG{LOG_FILE}\": $!", 1);
}
}
}
# Write pid file
if (open(OUT, ">$CONFIG{PID_DIR}/$PID_FILE")) {
print OUT "$$";
close(OUT);
}
# Move to the last position
if ($OLD_OFFSET && ($CONFIG{LOG_FILE} !~ /\.gz/)) {
if ((lstat($CONFIG{LOG_FILE}))[7] < $OLD_OFFSET) {
&dprint("Log file may have changed, rereading from start of the log file.") if ($CONFIG{DEBUG} > 0);
$OLD_OFFSET = 0;
} else {
&dprint("Jumping to last log offset ($OLD_OFFSET).") if ($CONFIG{DEBUG} > 0);
my $ret = seek(SA_FILE, $OLD_OFFSET, 0);
if (!$ret) {
&dprint("Wrong offset reread from start of the log file.") if ($CONFIG{DEBUG} > 0);
seek(SA_FILE, 0, 0);
$OLD_OFFSET = 0;
}
# Exceptional case of a file with same size, read the first line
# to see if the data has changed, otherwise this is the same file
if ($OLD_OFFSET && (lstat($CONFIG{LOG_FILE}))[7] == $OLD_OFFSET) {
# Rewind at begining of the line
my $c = '';
my $count = 1;
while ($c ne "\n" && $count < 10000) {
$count++;
seek(SA_FILE, $OLD_OFFSET - $count, 0);
read(SA_FILE, $c, 1);
}
seek(SA_FILE, $OLD_OFFSET - $count, 0);
if ($count < 10000) {
while (my $l = <SA_FILE>) {
chomp($l);
$l =~ s/ ID \d+ mail.\w//;
next if ($l =~ /policy-spf|You are still greylisted/);
my $tmp_last_parsed = $l;
# Only catch relevant logs
next if ($CONFIG{EXCLUDE_LINE} && $tmp_last_parsed =~ m#$CONFIG{EXCLUDE_LINE}#);
if ( ($tmp_last_parsed =~ /($CONFIG{MTA_NAME}|$CONFIG{MAILSCAN_NAME}|$CONFIG{AMAVIS_NAME}|$CONFIG{MD_NAME}|$CONFIG{CLAMD_NAME}|$CONFIG{POSTGREY_NAME}|$CONFIG{SPAMD_NAME}|$CONFIG{CLAMSMTPD_NAME}|$CONFIG{SPF_DKIM_NAME})[\/\[:]/) || ($LAST_PARSED =~ $EXIM_REGEX) ) {
if ($tmp_last_parsed ne $OLD_LAST_PARSED) {
&dprint("Size is identique but data are more recent than the one at old offset. Rereading from start of the log file.") if ($CONFIG{DEBUG} > 0);
seek(SA_FILE, 0, 0);
$OLD_OFFSET = 0;
$OLD_LAST_PARSED = '';
} else {
&dprint("Size is identique and data are the same, nothing to do.") if ($CONFIG{DEBUG} > 0);
# Go back to current offset
seek(SA_FILE, $OLD_OFFSET, 0);
}
last;
}
}
} else {
# Go back to current offset
seek(SA_FILE, $OLD_OFFSET, 0);
}
}
}
} else {
$OLD_OFFSET = 0;
}
while (my $l = <SA_FILE>) {
chomp($l);
$l =~ s/ ID \d+ mail.\w//;
next if ($l =~ /policy-spf|You are still greylisted/);
$LAST_PARSED = $l;
$l = '';
# Only catch relevant logs
next if ($CONFIG{EXCLUDE_LINE} && $LAST_PARSED =~ m#$CONFIG{EXCLUDE_LINE}#);
if ( ($LAST_PARSED =~ /($CONFIG{MTA_NAME}|$CONFIG{MAILSCAN_NAME}|$CONFIG{AMAVIS_NAME}|$CONFIG{MD_NAME}|$CONFIG{CLAMD_NAME}|$CONFIG{POSTGREY_NAME}|$CONFIG{SPAMD_NAME}|$CONFIG{CLAMSMTPD_NAME}|$CONFIG{SPF_DKIM_NAME})[\/\[:]/) || ($LAST_PARSED =~ $EXIM_REGEX) ) {
my $tmpos = tell(SA_FILE);
if ($OLD_LAST_PARSED) {
# Store the last position in the log
# Line already parsed ? If yes, go to retrieve next log line
if (&incremental_check($LAST_PARSED, $OLD_LAST_PARSED, $OLD_OFFSET) != 0) {
# Search if this is really the same log file
if ($OLD_OFFSET && ($tmpos > $OLD_OFFSET)) {
&dprint("Data are more recent than the one at old offset, maybe the file has changed. Rereading from start of the log file.") if ($CONFIG{DEBUG} > 0);
seek(SA_FILE, 0, 0);
$OLD_OFFSET = 0;
$OLD_LAST_PARSED = '';
}
next;
}
# The line have not already been parsed so erase last date
# to not go back to this block again
$OLD_LAST_PARSED = '';
}
if ($CONFIG{LOG_FILE} !~ /\.gz/) {
$OLD_OFFSET = $tmpos;
}
# Extract common fields, store data in memory and retrieve current time
my $check_time = &store_data(&parse_common_fields(split(/\s+/, $LAST_PARSED)));
# Flush data to disk each kind of 5 seconds at least by default
if ($check_time > $CURRENT_TIME+$CONFIG{DELAY}) {
$CURRENT_TIME = $check_time;
&dprint("Flushing data to disk...");
&flush_data();
}
}
}
&dprint("Flushing data to disk...");
&flush_data();
if ($CONFIG{BREAK}) {
unlink("$CONFIG{PID_DIR}/$PID_FILE");
exit 0;
}
}
# Daemon mode is not possible with compressed log file
if ($CONFIG{LOG_FILE} =~ /\.gz/) {
&dprint("Daemon mode is not possible with compressed log file", 1);
}
# Open a pipe to the tail program or to the journalctl command
my $since = '';
if ($CONFIG{JOURNALCTL_CMD} !~ /--since|-S/) {
if ($LAST_PARSED) {
if ($LAST_PARSED =~ /^(\d+)-(\d+)-(\d+).(\d+):(\d+):(\d+)/) {
$since = " --since=\"$1-$2-$3 $4:$5:$6\"";
}
} else {
if ($OLD_LAST_PARSED =~ /^(\d+)-(\d+)-(\d+).(\d+):(\d+):(\d+)/) {
$since = " --since=\"$1-$2-$3 $4:$5:$6\"";
}
}
}
if ($CONFIG{JOURNALCTL_CMD}) {
&dprint("Opening pipe to command: $CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\" -f");
if (!($KID = $SA_PIPE->open("$CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\" -f |"))) {
&dprint("$0: cannot read input from command: $CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\" -f, $!", 1);
}
} else {
&dprint("Opening pipe to $CONFIG{TAIL_PROG} $CONFIG{TAIL_ARGS} $CONFIG{LOG_FILE}");
if ( !($KID = $SA_PIPE->open("$CONFIG{TAIL_PROG} $CONFIG{TAIL_ARGS} $CONFIG{LOG_FILE}|"))) {
&dprint("$0: cannot read from pipe to \"$CONFIG{TAIL_PROG} $CONFIG{TAIL_ARGS} $CONFIG{LOG_FILE}\": $!");
}
}
# Write pid file
if (open(OUT, ">$CONFIG{PID_DIR}/$PID_FILE")) {
print OUT "$$";
close(OUT);
}
# We'll need a non blocking read to be able to intercept signal
# when there's no new entry in the log file. But for now
# SIGTERM should be use to interrupt the blocking read.
while (my $l = <$SA_PIPE>) {
chomp($l);
$l =~ s/ ID \d+ mail.\w//;
next if ($l =~ /policy-spf|You are still greylisted/);
$LAST_PARSED = $l;
$l = '';
# Only catch relevant logs
next if ($CONFIG{EXCLUDE_LINE} && $LAST_PARSED =~ m#$CONFIG{EXCLUDE_LINE}#);
my $check_time = '';
if ( ($LAST_PARSED =~ /($CONFIG{MTA_NAME}|$CONFIG{MAILSCAN_NAME}|$CONFIG{AMAVIS_NAME}|$CONFIG{MD_NAME}|$CONFIG{CLAMD_NAME}|$CONFIG{POSTGREY_NAME}|$CONFIG{SPAMD_NAME}|$CONFIG{CLAMSMTPD_NAME}|$CONFIG{SPF_DKIM_NAME})[\/\[]/) || ($LAST_PARSED =~ $EXIM_REGEX) ) {
# Extract common fields and store data in memory
$check_time = &store_data(&parse_common_fields(split(/\s+/, $LAST_PARSED)));
} else {
$check_time = &format_time(localtime(time));
}
# Flush data to disk if write delay is over
if ($check_time > $CURRENT_TIME+$CONFIG{DELAY}) {
$CURRENT_TIME = $check_time;
&dprint("Flushing data to disk ($check_time > $CURRENT_TIME+$CONFIG{DELAY})...");
&flush_data();
# Avoid memory leak
%POSTFIX_PLUGIN_TEMP_DELIVERY = ();
%AMAVIS_ID = ();
}
}
&dprint("Flushing last data to disk...");
&flush_data(1);
}
####
# Routine used to extract common field on maillog lines
####
sub parse_common_fields
{
my ($month,$day,$time,$host,$type,@other) = @_;
# Get current system time
my $ctime = &format_time(localtime(time));
# Remove domain part of the host
if ($CONFIG{NO_HOST_DOMAIN}) {
$host =~ s/\..*//;
}
my $date = '';
if ($month =~ /(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)/) {
$date = "$1$2$3";
unshift(@other, $type);
unshift(@other, $host);
$host = $day;
$type = $time;
$time = "$4$5$6";
} elsif ($month =~ /(\d+)-(\d+)-(\d+)/) {
$date = "$1$2$3";
unshift(@other, $type);
$type = $host;
$host = $time;
if ($day =~ /(\d+):(\d+):(\d+)/) {
$time = "$1$2$3";
}
} else {
my $current_year = 0;
if ($CONFIG{DEFAULT_YEAR}) {
$current_year = $CONFIG{DEFAULT_YEAR};
} else {
$current_year = (localtime(time))[5]+1900;
}
$date = $current_year . sprintf("%02d",$MONTH_TO_NUM{"$month"}) . sprintf("%02d",$day);
my @f = split(/:/, $time);
$f[0] = sprintf("%02d",$f[0]);
$f[1] = sprintf("%02d",$f[1]);
$f[2] = sprintf("%02d",$f[2]);
$time = "$f[0]$f[1]$f[2]";
if ("$date$time" > $ctime) {
# If log timestamp is in the future, use the given one
if (!$CONFIG{DEFAULT_YEAR}) {
$date = ($current_year - 1) . sprintf("%02d",$MONTH_TO_NUM{"$month"}) . sprintf("%02d",$day);
}
}
}
$type =~ s/\[.*\]\://;
$host = $CONFIG{MERGING_HOST} if ($CONFIG{MERGING_HOST});
my $line = join(' ', @other);
$line =~ s/^\[ID.*\] //;
return ($date,$time,$host,$type,$line,$ctime);
}
####
# Routine used to store collected data
####
sub store_data
{
my ($date,$time,$host,$type,$other,$ctime) = @_;
if ($type =~ /^[<=>\*\-]+$/) {
&parse_exim("$date","$time",$host,$type,$other);
} elsif (($CONFIG{MAILSCAN_NAME} || $CONFIG{CLAMD_NAME}) && ($type =~ /^$CONFIG{MAILSCAN_NAME}|$CONFIG{CLAMD_NAME}/i)) {
&parse_mailscanner("$date","$time",$host,$other);
} elsif ($CONFIG{AMAVIS_NAME} && ($type =~ /^$CONFIG{AMAVIS_NAME}/i)) {
&parse_amavis("$date","$time",$host,$other);
} elsif ($CONFIG{CLAMSMTPD_NAME} && ($type =~ /^$CONFIG{CLAMSMTPD_NAME}/i)) {
&parse_clamsmtpd("$date","$time",$host,$other);
} elsif ($CONFIG{MTA_NAME} && ($type =~ /^$CONFIG{MTA_NAME}/i)) {
&parse_sendmail("$date","$time",$host,$other,$type);
} elsif ($CONFIG{MD_NAME} && ($type =~ /^$CONFIG{MD_NAME}/i)) {
&parse_mimedefang("$date","$time",$host,$other);
} elsif ($CONFIG{POSTGREY_NAME} && ($type =~ /^$CONFIG{POSTGREY_NAME}/i)) {
&parse_postgrey("$date","$time",$host,$other);
} elsif ($CONFIG{SPAMD_NAME} && ($type =~ /^$CONFIG{SPAMD_NAME}/i)) {
&parse_spamd("$date","$time",$host,$other);
} elsif ($CONFIG{SPF_DKIM_NAME} && ($type =~ /^$CONFIG{SPF_DKIM_NAME}/i)) {
&parse_spf_dkim("$date","$time",$host,$other);
} else {
&dprint("Skipping unknown syslog report => $date $time $host [$type]: $other") if ($CONFIG{DEBUG} > 1);
}
return $ctime;
}
sub parse_exim
{
my ($date,$time,$id,$type,$other) = @_;
my $time_st = "$date$time";
my $host = $HOSTNAME || 'unknown';
if ($type eq '<=') {
if ($other =~ m#^(.*?) H=(.*?) P=.* S=(\d+) id=(.*)#) {
my $size = $3;
my $msgid = $4;
my $relay = &clean_relay(lc($2));
$FROM{$host}{$id}{from} = &edecode($1);
$msgid =~ s/[<>]//g;
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{size} = $size;
$FROM{$host}{$id}{nrcpts} = 1;
$FROM{$host}{$id}{msgid} = $msgid;
$FROM{$host}{$id}{relay} = $relay;
}
} elsif ($type =~ /^(=|-)>$/) {
if ($other =~ m#^(.*?) R=(.*?) T=(.*?) H=(.*)#) {
my $to = &edecode($1);
my $relay = &clean_relay(lc($4));
if ($relay eq $CONFIG{'SKIP_RCPT_RELAY'}) {
return;
}
next if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{relay}}, $relay);
push(@{$TO{$host}{$id}{to}}, $to);
push(@{$TO{$host}{$id}{status}}, 'Sent');
}
} elsif ($type eq '==') {
if ($other =~ m#^(.*?) R=(.*?) T=([^:]+): (.*)#) {
$REJECT{$host}{$id}{relay} = $FROM{$host}{$id}{relay};
$REJECT{$host}{$id}{status} = &clear_status($2);
$REJECT{$host}{$id}{date} = $time_st;
$REJECT{$host}{$id}{arg1} = &edecode($1);
my $rule = $3;
$rule =~ s/\s*\(.*//;
$REJECT{$host}{$id}{rule} = $rule;
}
} elsif ($type eq '**') {
if ( !exists $REJECT{$host}{$id}{rule} && ($other =~ m#^([^:]+): (.*)#) ) {
$REJECT{$host}{$id}{relay} = $FROM{$host}{$id}{relay};
$REJECT{$host}{$id}{rule} = 'error';
$REJECT{$host}{$id}{status} = &clear_status($2);
$REJECT{$host}{$id}{date} = $time_st;
$REJECT{$host}{$id}{arg1} = &edecode($1);
}
}
}
####
# Parse Sendmail syslog output
####
sub parse_sendmail
{
my ($date,$time,$host,$str,$type) = @_;
my $time_st = "$date$time";
# Skip unwanted Postfix lines
if ($str =~ m#^([^:\s]+): .*#) {
my $id = $1;
return if ($id =~ /^(warning|(match|dns|rewrite|generic|mail|maps|ctable|smtp)_.*)$/); # Skip debug message
return if (exists $SKIPMSG{$id});
}
#### Store each relevant information per host and id
# Some spampd line must be skipped
return if (($type eq 'spampd') && ($str =~ /(processing|clean) message/));
# Parse MTA system error
if ($str =~ m#^([^:\s]+): SYSERR[^:]+: (.*)# ) {
$SYSERR{$host}{$1}{date} = $time_st;
$SYSERR{$host}{$1}{message} = &clear_status($2);
# Skip message related to MCI caching module
} elsif ($str =~ m#^([^:\s]+): MCI\@#) {
return;
# Skip Debug message
} elsif ($str =~ m#^([^:\s]+):\s+\d+: fl=#) {
return;
# POSTFIX: Skip connect/disconnect message
} elsif ($str =~ m#^(DIS)?CONNECT #i) {
return;
# POSTFIX temporary blacklist/whitelist messsage
} elsif ($str =~ m#^(PASS OLD|PASS NEW|WHITELISTED|BLACKLISTED)#i) {
return;
# POSTFIX: Skip postscreen messages
} elsif ($str =~ m#^(WHITELIST VETO|BARE NEWLINE)#i) {
return;
# POSTFIX pregreet test
} elsif ($str =~ m#^(PREGREET|HANGUP)#i) {
return;
# POSTFIX dnsbl message
} elsif ($str =~ m#^DNSBL rank#i) {
return;
# Debug and info messages from POSTFIX
} elsif ($str =~ /^(DEBUG|INFO) /) {
return;
# POSFIX TLS connexion
} elsif ($str =~ /(connect from|setting up TLS connection from)/) {
return;
} elsif ($str =~ /(connect to|setting up TLS connection to|Untrusted TLS connection established)/) {
return;
# POSTFIX dnsbl message ???
} elsif ($str =~ m#addr [a-fA-F0-9\.\:]+ listed#) {
return;
# POSTFIX postscreen messages: COMMAND (PIPELINING|COUNT LIMIT|TIME LIMIT)???
} elsif ($str =~ m#^COMMAND #i) {
return;
# POSTFIX: error messages
} elsif ($str =~ m#^(lost connection|timeout|too many errors) after ([^\s]+)#) {
$SYSERR{$host}{"$date$time"}{date} = $time_st;
$SYSERR{$host}{"$date$time"}{message} = $1 . ' after ' . $2;
} elsif ($str =~ m#^connect to [^:]+: (.*)#) {
$SYSERR{$host}{"$date$time"}{date} = $time_st;
$SYSERR{$host}{"$date$time"}{message} = $1;
} elsif ($str =~ m#^(certificate verification failed for).*:( untrusted issuer| self-signed certificate).*#) {
$SYSERR{$host}{"$date$time"}{date} = $time_st;
$SYSERR{$host}{"$date$time"}{message} = $1 . $2;
} elsif ($str =~ m#^(SSL_connect error).*#) {
$SYSERR{$host}{"$date$time"}{date} = $time_st;
$SYSERR{$host}{"$date$time"}{message} = $1;
# Sendmail Milter change subject
} elsif ($str =~ m#^([^:\s]+): Milter change: header Subject: from (.*?) to (.*)$#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
$FROM{$host}{$id}{subject} = &decode_subject($2);
# Postfix subject information
} elsif ($str =~ m#^([^:\s]+): (warning|info): header Subject: (.*?) from ([^;]+);#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
$FROM{$host}{$id}{subject} = &decode_subject($3);
} elsif ($str =~ m#^warning: .*#) {
$SYSERR{$host}{"$date$time"}{date} = $time_st;
$SYSERR{$host}{"$date$time"}{message} = &clear_status($str);
# POSTFIX spampd pass
} elsif ($str =~ m#identified spam (<[^>]+>) \(([^\)]+)\) from ([^\s]+) for ([^\s]+) .* (\d+) bytes.#) {
my $id = $1;
my $rule = 'spampd';
my $from = &edecode($3);
my $to = &edecode($4);
my $status = $2;
my $size = $5;
foreach my $i (keys %MSGID) {
if ($i eq $id) {
$id = $MSGID{$i}{id};
last;
}
}
return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
$SPAM{$host}{$id}{relay} = $FROM{$host}{$id}{relay};
$SPAM{$host}{$id}{rule} = 'reject';
$SPAM{$host}{$id}{spam} = $status;
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{from} = $from;
$SPAM{$host}{$id}{to} = $to;
$SPAM{$host}{$id}{status} = $status;
# POSTFIX spampd pass
} elsif ($str =~ m#identified spam \(([^\)]+)\) \(([^\)]+)\) from ([^\s]+) for ([^\s]+) .* (\d+) bytes.#) {
my $id = $1;
my $rule = 'spampd';
my $from = &edecode($3);
my $to = &edecode($4);
my $status = $2;
my $size = $5;
return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
$status =~ s/.*\///;
$SPAMPD{$host}{$from}{$to}{rule} = 'reject';
$SPAMPD{$host}{$from}{$to}{spam} = $status;
$SPAMPD{$host}{$from}{$to}{date} = $time_st;
$SPAMPD{$host}{$from}{$to}{status} = $status;
# Sendmail authid single message
} elsif ($str =~ m#^([^:\s]+): authid=#) {
return;
# POSTFIX remove message id
} elsif ($str =~ m#^([^:\s]+): removed$#) {
my $id = $1;
foreach (keys %MSGID) {
delete $MSGID{$_} if ($MSGID{$_}{id} eq $id);
}
delete $SKIPMSG{$id};
delete $KEEP_TEMPORARY{$id};
return;
# Sendmail subject information
} elsif ($str =~ m#^([^:\s]+): Subject:(.*)#) {
my $id = $1;
$FROM{$host}{$id}{subject} = &decode_subject($2);
# Parse protocole error
} elsif ($str =~ m#^([^:\s]+): ([^:\s]+): (.*protocol error:.*)#) {
$SYSERR{$host}{$1}{date} = $time_st;
$SYSERR{$host}{$1}{message} = $3;
# Parse virus found by clamav-milter
} elsif ($str =~ m#^([^:\s]+): Milter add: header: X-Virus-Status: Infected with (.*)#) {
$VIRUS{$host}{$1}{virus} = $2;
$VIRUS{$host}{$1}{file} = 'Inline';
$VIRUS{$host}{$1}{date} = $time_st;
} elsif ($str =~ m#^([^:\s]+): Milter [^:]+: header: X-Virus-Status: Infected \((.*)\)#) {
$VIRUS{$host}{$1}{virus} = $2;
$VIRUS{$host}{$1}{file} = 'Inline';
$VIRUS{$host}{$1}{date} = $time_st;
# Parse spam found by spamd-milter
} elsif ($str =~ m#^([^:\s]+): Milter (add|change): header: X-Spam-Status: Yes, score=([^\s]+) required=([^\s]+) tests=([^\s]+)#) {
my $id = $1;
$SPAM{$host}{$id}{spam} = 'spamdmilter';
$SPAM{$host}{$id}{date} = $time_st;
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'spamdmilter';
$SPAMDETAIL{$host}{$id}{spam} = $5;
$SPAMDETAIL{$host}{$id}{required} = $4;
$SPAMDETAIL{$host}{$id}{score} = $3;
if ($SPAMDETAIL{$host}{$id}{spam} =~ s/(nt|\s)autolearn=([^\s,]+)//) {
$SPAMDETAIL{$host}{$id}{autolearn} = $2;
}
$SPAMDETAIL{$host}{$id}{spam} =~ s/,nt//g;
}
# Parse spam found by postfix/policyd-weight
} elsif ($str =~ m# action=DUNNO\s+(.*); rate: ([^;]+); <client=([^>]+)> <helo=([^>]+)> <from=([^>]+)> <to=([^>]+)>; delay.*#) {
my $spam = $1;
my $score = $2;
my $relay = $3;
my $from = $5;
my $to = $6;
my $id = &get_uniqueid();
$SPAM{$host}{$id}{spam} = 'policydweight';
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{relay} = &clean_relay($relay);
$SPAM{$host}{$id}{from} = &edecode($from);
$SPAM{$host}{$id}{to} = &edecode($to);
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'policydweight';
$SPAMDETAIL{$host}{$id}{spam} = $spam;
$SPAMDETAIL{$host}{$id}{score} = $score;
}
# Parse spam found in cache by postfix/policyd-weight
} elsif ($str =~ m# action=DUNNO\s+(.*); rate: ([^;]+); <client=([^>]+)> <helo=([^>]+)> <from=([^>]+)> <to=([^>]+)>; delay.*#) {
my $spam = $1;
my $score = $2;
my $relay = $3;
my $from = $5;
my $to = $6;
my $id = &get_uniqueid();
$SPAM{$host}{$id}{spam} = 'policydweight';
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{relay} = &clean_relay($relay);
$SPAM{$host}{$id}{from} = &edecode($from);
$SPAM{$host}{$id}{to} = &edecode($to);
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'policydweight';
$SPAMDETAIL{$host}{$id}{spam} = $spam;
$SPAMDETAIL{$host}{$id}{score} = $score;
}
# Parse spam found by dnsbl-milter
} elsif ($str =~ m#^([^:\s]+): Milter: from=([^,]+), reject=(.*)#) {
my $id = $1;
my $spam = $3;
$spam =~ s/ See .*//;
$spam =~ s/\/.*//;
$SPAM{$host}{$id}{spam} = 'dnsblmilter';
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{from} = &edecode($2);
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'dnsblmilter';
$SPAMDETAIL{$host}{$id}{spam} = $spam;
}
# Parse spam found by jchkmail
} elsif ($str =~ m#^([^:\s]+): Milter (add|change): header: X-j-chkmail-Status: (Spam|Unsure)(.*)#) {
$SPAM{$host}{$1}{spam} = 'jchkmail';
$SPAM{$host}{$1}{date} = $time_st;
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$1}{type} = 'jchkmail';
$SPAMDETAIL{$host}{$1}{spam} = $3 . $4;
$SPAMDETAIL{$host}{$1}{date} = $time_st;
}
} elsif ($str =~ m#^([^:\s]+): Milter (add|change): header: X-j-chkmail-Score: .*> S=(.*)#) {
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$1}{type} = 'jchkmail';
$SPAMDETAIL{$host}{$1}{score} = $3;
}
} elsif ($str =~ m#^([^:\s]+): Milter (delete|add|change): #) {
# Skip other milter header modication notice
return;
} elsif ($str =~ m#^([^:\s]+): Milter insert \(\d+\): #) {
# Skip other milter header insertion notice
return;
} elsif ($str =~ m#^([^:\s]+): Milter: connect: .*, ([^,]*)#) {
$SYSERR{$host}{$1}{date} = $time_st;
$SYSERR{$host}{$1}{message} = &clear_status($2);
} elsif ($str =~ m#^([^:\s]+): Milter (delete|add|change) #) {
# Skip other milter modication notice
return;
} elsif ($str =~ m#^([^:\s]+): Milter: data, reject=(.*)#) {
my $id = $1;
my $status = $2;
if ($status =~ /Blocked by SpamAssassin/) {
$SPAM{$host}{$id}{spam} = 'Blocked by SpamAssassin';
} elsif ($status =~ /554 5\.7\.1 (.*) detected by ClamAV.*/) {
$SPAM{$host}{$id}{spam} = "ClamAv $1"
} elsif ($status =~ /550 5\.7\.0 (.*) id=/) {
$SPAM{$host}{$id}{spam} = "eXpurGate $1"
} elsif ($status =~ /554 5\.7\.1 Command rejected/) {
return # Already handle at Clamav virus report
} elsif ($status =~ /Please try again later/) {
return; # Skip greylist rejection
} else {
$SPAM{$host}{$id}{spam} = $status;
}
# Local failure
} elsif ($str =~ m#^([^:\s]+): <([^>]+)>\.\.\. (.*)#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
my $to = $2;
my $status = &clear_status($3);
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
$to = &edecode($to);
return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{to}}, $to);
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{status}}, $status);
# POSTFIX queue From clause
} elsif ($str =~ m#^([^:\s]+): from=([^,]*), size=(\d+), nrcpt=(\d+).*#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
my $from = lc($2);
$from ||= 'empty';
my $size = $3;
my $nrcpts = $4;
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{from} = &edecode($from);
$FROM{$host}{$id}{size} = $size;
$FROM{$host}{$id}{nrcpts} = $nrcpts;
if (exists $SPAMPD{$host}{$from}) {
foreach my $to (keys %{$SPAMPD{$host}{$from}}) {
$SPAM{$host}{$id}{relay} = $FROM{$host}{$id}{relay};
$SPAM{$host}{$id}{rule} = $SPAMPD{$host}{$from}{$to}{rule};
$SPAM{$host}{$id}{spam} = $SPAMPD{$host}{$from}{$to}{spam};
$SPAM{$host}{$id}{date} = $SPAMPD{$host}{$from}{$to}{date};
$SPAM{$host}{$id}{status} = $SPAMPD{$host}{$from}{$to}{status};
}
delete $SPAMPD{$host}{$from};
}
# Catch POSTFIX SASL AUTH method
} elsif ($str =~ m#^([^:\s]+): client=([^,]+), sasl_method=([^,]+), sasl_username=(.*)#) {
my $id = $1;
my $authid = $4;
my $relay = &clean_relay(lc($2));
push(@{$AUTH{$host}{$authid}{type}}, 'SASL');
push(@{$AUTH{$host}{$authid}{mech}}, $3);
push(@{$AUTH{$host}{$authid}{date}}, $time_st);
push(@{$AUTH{$host}{$authid}{relay}}, $relay);
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{relay} = $relay;
# POSTFIX client origin and relay
} elsif ($str =~ m#^([^:\s]+): client=([^,]*)#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
my $relay = &clean_relay(lc($2));
# POSTFIX origin queue
# Get real origin when available
if ($str =~ m#^[^:\s]+: client=([^,]+), orig_queue_id=([^,]+), orig_client=([^,]+)#) {
$KEEP_TEMPORARY{$id} = $2;
$id = $2;
$relay = &clean_relay(lc($3));
}
$FROM{$host}{$id}{relay} = $relay;
# Catch POSTFIX SASL SMTP AUTH
if ($str =~ m#sasl_method=([^,]*), sasl_username=([^,]*)#) {
my $authid = $2;
push(@{$AUTH{$host}{$authid}{type}}, 'SASL');
push(@{$AUTH{$host}{$authid}{mech}}, $1);
push(@{$AUTH{$host}{$authid}{date}}, $time_st);
push(@{$AUTH{$host}{$authid}{relay}}, $relay);
}
# POSTFIX message id
} elsif ($str =~ m#^([^:\s]+): message-id=([^,]*)#) {
next if ($1 eq '<>');
my $id = $KEEP_TEMPORARY{$1} || $1;
my $msgid = $2;
$msgid =~ s/[<>]//g;
if (exists $MSGID{$msgid}) {
$KEEP_TEMPORARY{$id} = $MSGID{$msgid}{id};
}
$MSGID{$msgid}{id} = $id;
$FROM{$host}{$id}{msgid} = $msgid;
return;
# POSTFIX local relay
} elsif ($str =~ m#^([^:\s]+): uid=\d+ from=#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
$FROM{$host}{$id}{relay} = 'localhost';
# From clause
} elsif ($str =~ m#^([^:\s]+): from=([^,]*), size=(\d+),.*, nrcpts=(\d+), msgid=([^,]+),.*relay=([^,]+)#) {
my $id = $1;
my $from = lc($2);
$from ||= 'empty';
my $size = $3;
my $nrcpts = $4;
my $msgid = $5;
my $relay = &clean_relay(lc($6));
$msgid =~ s/[<>]//g;
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{from} = &edecode($from);
$FROM{$host}{$id}{size} = $size;
$FROM{$host}{$id}{nrcpts} = $nrcpts;
$FROM{$host}{$id}{msgid} = $msgid;
$FROM{$host}{$id}{relay} = $relay;
} elsif ($str =~ m#^([^:\s]+): from=([^,]*), size=(\d+),.*, nrcpts=(\d+),.*relay=([^,]+)#) {
my $id = $1;
my $from = lc($2);
$from ||= 'empty';
my $size = $3;
my $nrcpts = $4;
my $relay = &clean_relay(lc($5));
if (exists $GREYLIST{$id}) {
delete $GREYLIST{$id};
return;
} elsif (exists $REJECT{$host}{$id} && !$size) {
$REJECT{$host}{$id}{arg1} = &edecode($from);
$REJECT{$host}{$id}{relay} = $relay;
}
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{from} = &edecode($from);
$FROM{$host}{$id}{size} = $size;
$FROM{$host}{$id}{nrcpts} = $nrcpts;
$FROM{$host}{$id}{relay} = $relay;
# Postfix cleanup entry
} elsif ($str =~ m#^([^:\s]+): hold: header Received: from ([^\)]*)\).* from=<([^>]+)> to=<([^>]+)>#) {
my $id = $1;
my $from = lc($3);
$from ||= 'empty';
my $to = lc($4);
my $relay = &clean_relay(lc($2));
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{from} = &edecode($from);
$FROM{$host}{$id}{size} = 0;
$FROM{$host}{$id}{nrcpts} = 1;
$FROM{$host}{$id}{relay} = $relay;
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{to}}, $to);
# POSTFIX To clause
} elsif ($str =~ m#^([^:\s]+): to=([^,]+), relay=([^,]+),.*status=(.*)#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
my $to = &edecode($2);
my $relay = &clean_relay(lc($3));
my $status = $4;
if (!$CONFIG{NO_QUEUE_EXCLUSION} && ($status =~ /queued as ([^\)]+)\)/ && $relay eq 'localhost')) {
$KEEP_TEMPORARY{$1} = $id;
return;
}
# Spam message discarded by amavisd and reported as sent must be affected to spam
if ($status =~ /sent \(250 [^,]+, ([^,]+), .* spam\)/) {
$status = ucfirst($1);
$SPAM{$host}{$id}{relay} = $relay;
$SPAM{$host}{$id}{rule} = 'discarded';
$SPAM{$host}{$id}{spam} = "Amavis $status spam";
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{from} = $FROM{$host}{$id}{from} if (exists $FROM{$host}{$id}{from});
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'amavis';
$SPAMDETAIL{$host}{$id}{spam} = $status;
}
return;
}
# Virus message discarded by amavisd and reported as sent must be affected to virus
if ($status =~ /sent \(250 [^,]+, ([^,]+), .* INFECTED: (.*)\)/) {
$status = ucfirst($1);
$VIRUS{$host}{$id}{file} = 'Inline';
$VIRUS{$host}{$id}{virus} = $2;
$VIRUS{$host}{$id}{relay} = $relay;
$VIRUS{$host}{$id}{from} = $FROM{$host}{$id}{from} if (exists $FROM{$host}{$id}{from});
$VIRUS{$host}{$id}{to} = $to;
$VIRUS{$host}{$id}{date} = $time_st;
return;
}
if ($status =~ /sent \(250 OK, sent [^\s]+ ([^\)]+)\)/) {
$KEEP_TEMPORARY{$1} = $id;
return;
}
$status = &clear_status($status, $id);
if ($relay eq $CONFIG{'SKIP_RCPT_RELAY'}) {
return;
}
# Store DSN id mapping
if ($status eq 'Bounced') {
$DSN{$host}{$id}{srcid} = $id;
$DSN{$host}{$id}{status} = $status;
$DSN{$host}{$id}{date} = $time_st;
} else {
$status = 'Sent' if ($status eq 'sent');
}
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
foreach my $t (split(/,/, $to)) {
$t = &edecode($t);
return if ($t eq 'more');
next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{relay}}, $relay);
push(@{$TO{$host}{$id}{to}}, $t);
push(@{$TO{$host}{$id}{status}}, $status);
}
# POSTFIX To clause with origine
} elsif ($str =~ m#^([^:\s]+): to=([^,]+), orig_to=([^,]*), relay=([^,]+),.*status=(.*)#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
my $to = &edecode($2);
my $relay = &clean_relay(lc($4));
if ($relay eq $CONFIG{'SKIP_RCPT_RELAY'}) {
return;
}
my $status = &clear_status($5, $id);
$status = 'Sent' if ($status eq 'sent');
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
foreach my $t (split(/,/, $to)) {
$t = &edecode($t);
return if ($t eq 'more');
next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{to}}, $t);
push(@{$TO{$host}{$id}{status}}, $status);
push(@{$TO{$host}{$id}{relay}}, $relay);
}
# To clause queued
} elsif ($str =~ m#^([^:\s]+): to=(.*), delay=.* stat=queued#) {
my $id = $1;
my $to = &edecode($2);
return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{queue_date}}, $time_st);
push(@{$TO{$host}{$id}{queue_to}}, $to);
# To clause generated by a mailing list
} elsif ($str =~ m#^([^:\s]+): to=(.*), ctladdr=([^\s]+).*, mailer=prog,.*stat=(.*)#) {
my $id = $1;
my $prog = lc($2);
my $ctladdr = &edecode($3);
my $status = &clear_status($4);
$prog =~ s/\|//g;
# Skip vacation and procmail prog that double the recipient entry
return if ($prog =~ /(vacation|procmail)/);
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
return if ($CONFIG{EXCLUDE_TO} && ($ctladdr =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{relay}}, 'localhost');
push(@{$TO{$host}{$id}{to}}, $ctladdr);
push(@{$TO{$host}{$id}{status}}, $status);
} elsif ($str =~ m#^([^:\s]+): to=(.*), ctladdr=([^\s]+).*, mailer=\*file\*,.*, stat=(.*)#) {
my $id = $1;
my $prog = lc($2);
my $ctladdr = &edecode($3);
my $status = &clear_status($4);
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
return if ($CONFIG{EXCLUDE_TO} && ($ctladdr =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{relay}}, 'localhost');
push(@{$TO{$host}{$id}{to}}, $ctladdr);
push(@{$TO{$host}{$id}{status}}, $status);
} elsif ($str =~ m#^([^:\s]+): to=(.*), ctladdr=([^\s]+).*, mailer=local, .*, stat=(.*)#) {
my $id = $1;
my $to = &edecode($2);
my $ctladdr = &edecode($3);
my $status = &clear_status($4);
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
foreach my $t (split(/,/, $to)) {
$t = &edecode($t);
return if ($t eq 'more');
next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{relay}}, 'localhost');
push(@{$TO{$host}{$id}{to}}, $t);
push(@{$TO{$host}{$id}{status}}, $status);
}
# To clause generated by a redirection
} elsif ($str =~ m#^([^:\s]+): to=(.*), ctladdr=([^\s]+).*, mailer=.*, relay=([^,]+),.*stat=(.*)#) {
my $id = $1;
my $to = &edecode($2);
my $ctladdr = &edecode($3);
my $relay = &clean_relay(lc($4));
if ($relay eq $CONFIG{'SKIP_RCPT_RELAY'}) {
return;
}
my $status = &clear_status($5);
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
foreach my $t (split(/,/, $to)) {
$t = &edecode($t);
return if ($t eq 'more');
next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{relay}}, 'localhost');
push(@{$TO{$host}{$id}{to}}, $t);
push(@{$TO{$host}{$id}{status}}, $status);
}
# To close of intercepted virus by clamav-milter
} elsif ($str =~ m#^([^:\s]+): to=(.*), delay=.*, stat=(virus .*) detected by#) {
my $id = $1;
my $to = $2;
my $status = &clear_status($3);
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
foreach my $t (split(/,/, $to)) {
$t = &edecode($t);
return if ($t eq 'more');
next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{to}}, $t);
push(@{$TO{$host}{$id}{status}}, $status);
}
# To clause with local distribution
} elsif ($str =~ m#^([^:\s]+): to=(.*), delay=.*, mailer=local,.*, stat=(.*)#) {
my $id = $1;
my $to = $2;
my $status = &clear_status($3);
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
foreach my $t (split(/,/, $to)) {
$t = &edecode($t);
return if ($t eq 'more');
next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{relay}}, 'localhost');
push(@{$TO{$host}{$id}{to}}, $t);
push(@{$TO{$host}{$id}{status}}, $status);
}
# To clause
} elsif ($str =~ m#^([^:\s]+): to=(.*), delay=.*, relay=([^,]+),.*stat=(.*)#) {
my $id = $1;
my $to = $2;
my $relay = &clean_relay(lc($3));
my $status = &clear_status($4);
if ($relay eq $CONFIG{'SKIP_RCPT_RELAY'}) {
return;
}
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
foreach my $t (split(/,/, $to)) {
$t = &edecode($t);
return if ($t eq 'more');
next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{relay}}, $relay);
push(@{$TO{$host}{$id}{to}}, $t);
push(@{$TO{$host}{$id}{status}}, $status);
}
# To clause with no delivery. Most of the time follow a reject.
} elsif ($str =~ m#^([^:\s]+): to=(.*), delay=.*, pri=([^,]+), stat=(.*)#) {
my $id = $1;
my $to = $2;
my $status = &clear_status($4);
return if (exists $REJECT{$host}{$id});
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
foreach my $t (split(/,/, $to)) {
$t = &edecode($t);
return if ($t eq 'more');
next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{to}}, $t);
push(@{$TO{$host}{$id}{status}}, $status);
}
# Ruleset reject clause
} elsif ($str =~ m#^([^:\s]+): ruleset=([^,]+), arg1=([^,]+), relay=([^,]+),.*reject=(.*)#) {
my $id = $1;
my $rule = $2;
my $arg1 = $3;
my $relay = &clean_relay(lc($4));
my $reject = $5;
$arg1 =~ s/[<>]+//g;
# Test Sendmail DNSBL spam scan
if (($reject =~ /553 5\.3\.0/i) || ($reject =~ /550 5\.7\.1/i && $reject =~ / see[:\s]| listed/i)) {
$SPAM{$host}{$id}{relay} = $relay;
$SPAM{$host}{$id}{rule} = $rule;
$SPAM{$host}{$id}{spam} = 'DNSBL Spam blocked';
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{from} = $arg1;
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'dnsbl';
$reject =~ s/.*(Spam blocked see: )/$1/;
$reject =~ s/.*(Spam blocked .* found in .*)/$1/;
$reject =~ s/.*Rejected: .* (listed at [^\s]+).*/$1/;
$SPAMDETAIL{$host}{$id}{spam} = &clear_status($reject);
}
} else {
$REJECT{$host}{$id}{relay} = $relay;
$REJECT{$host}{$id}{rule} = $rule;
$REJECT{$host}{$id}{status} = &clear_status($reject);
$REJECT{$host}{$id}{date} = $time_st;
$REJECT{$host}{$id}{arg1} = &edecode($arg1);
}
# Ruleset reject clause
} elsif ($str =~ m#^ruleset=([^,]+), arg1=([^,]+),.* relay=([^,]+),.*reject=(.*)#) {
my $rule = $1;
my $arg1 = &clean_relay(lc($2));
my $relay = &clean_relay(lc($3));
my $reject = $4;
$arg1 =~ s/[<>]+//g;
my $id = &get_uniqueid();
# Test Sendmail DNSBL spam scan
if (($reject =~ /553 5\.3\.0/i) || ($reject =~ /550 5\.7\.1/i && $reject =~ / see[:\s]| listed/i)) {
$SPAM{$host}{$id}{relay} = $relay;
$SPAM{$host}{$id}{rule} = $rule;
$SPAM{$host}{$id}{spam} = 'DNSBL Spam blocked';
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{from} = $arg1;
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'dnsbl';
$reject =~ s/.*(Spam blocked see: )/$1/;
$reject =~ s/.*(Spam blocked .* found in .*)/$1/;
$SPAMDETAIL{$host}{$id}{spam} = $reject;
}
} else {
$REJECT{$host}{$id}{relay} = $relay;
$REJECT{$host}{$id}{rule} = $rule;
$REJECT{$host}{$id}{status} = &clear_status($reject);
$REJECT{$host}{$id}{date} = $time_st;
$REJECT{$host}{$id}{arg1} = &edecode($arg1);
}
# Add support to milter recipient rejection report
} elsif ($str =~ m#^([^:\s]+): Milter: to=(.*), reject=(\d+ \d+\.\d+\.\d+\s+.*)#) {
my $id = $1;
my $status = $3;
my $to = &edecode($2);
return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
if ($status =~ /Greylisting in action/) {
$GREYLIST{$id} = $to;
return;
}
$status = &clear_status($status);
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{to}}, $to);
push(@{$TO{$host}{$id}{status}}, $status);
$REJECT{$host}{$id}{rule} = 'reject';
$REJECT{$host}{$id}{status} = $status;
$REJECT{$host}{$id}{date} = $time_st;
# Add support to milter sender rejection
} elsif ($str =~ m#^([^:\s]+): Milter: from=(.*), reject=(\d+ \d+\.\d+\.\d+\s+.*)#) {
my $id = $1;
my $arg1 = $2;
my $reject = $3;
$arg1 =~ s/[<>]+//g;
if ($reject =~ /sender=(.*)\&ip=(.*)\&receiver=/) {
$REJECT{$host}{$id}{arg1} = &edecode($1);
$REJECT{$host}{$id}{relay} = $2;
} else {
$REJECT{$host}{$id}{arg1} = $arg1;
}
$REJECT{$host}{$id}{rule} = 'reject' if (!$REJECT{$host}{$id}{rule});
$REJECT{$host}{$id}{status} = &clear_status($reject);
$REJECT{$host}{$id}{date} = $time_st;
# Parse virus quarantined by clamav-milter
} elsif ($str =~ m#^([^:\s]+): milter=clamav-milter, quarantine=quarantined by clamav-milter#) {
if (!exists $VIRUS{$host}{$1}{virus}) {
$VIRUS{$host}{$1}{virus} = 'Quarantined by clamav-milter';
$VIRUS{$host}{$1}{file} = 'Inline';
$VIRUS{$host}{$1}{date} = $time_st;
} else {
$VIRUS{$host}{$1}{virus} .= ' - Quarantined by clamav-milter';
}
# Try to find milter rejection rule (other fields will be overriden by the condition above)
} elsif ($str =~ m#^([^:\s]+): milter=([^,]+), action=([^,]+), reject=(\d+ \d+\.\d+\.\d+\s+.*)#) {
my $id = $1;
my $reject = $4;
$REJECT{$host}{$id}{rule} = $2;
$REJECT{$host}{$id}{action} = $3;
$REJECT{$host}{$id}{status} = &clear_status($reject);
$REJECT{$host}{$id}{date} = $time_st;
if ($reject =~ /sender=(.*)\&ip=(.*)\&receiver=/) {
$REJECT{$host}{$id}{arg1} = &edecode($1);
$REJECT{$host}{$id}{relay} = $2;
}
# Store DSN id mapping
} elsif ($str =~ m#^([^:\s]+): ([^:\s]+): (DSN|return to sender|sender notify|postmaster notify): (.*)# ) {
my $previd = $1;
my $id = $2;
my $type = $3;
my $status = $4;
$status =~ s/(.*)\.\.\. //;
$DSN{$host}{$id}{srcid} = $previd;
$DSN{$host}{$id}{status} = &clear_status($type . " " . $status);
$DSN{$host}{$id}{date} = $time_st;
$DSN{$host}{$id}{status} =~ s/ \((.*?)\)//g;
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{from} = 'DSN@localhost';
$FROM{$host}{$id}{size} = 0;
$FROM{$host}{$id}{nrcpts} = 1;
$FROM{$host}{$id}{relay} = 'localhost';
# POSTFIX reject messages
} elsif ($str =~ m#^([^:\s]+): reject: RCPT from ([^\s]+) (.*) from=<([^>]*)>[,]* to=<([^>]+)>#) {
my $reject = $3;
my $from = $4;
my $to = $5;
my $relay = &clean_relay(lc($2));
my $id = &get_uniqueid();
my $status = '';
if ($reject =~ /^([^;]+)[;]/) {
$status = $1;
}
$reject =~ s/^\s+//;
$reject =~ s/[\s;]+$//;
# Test PostFix DNSBL spam scan
if ($reject =~ /(?:client|helo) .* (blocked using .*)/i) {
my $spamdetail = $1;
$SPAM{$host}{$id}{relay} = $relay;
$SPAM{$host}{$id}{rule} = 'reject';
$SPAM{$host}{$id}{spam} = 'DNSBL Spam blocked';
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{from} = &edecode($from);
$SPAM{$host}{$id}{to} = &edecode($to);
$SPAM{$host}{$id}{status} = &clear_status($status);
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'dnsbl';
$SPAMDETAIL{$host}{$id}{spam} = &clear_status($spamdetail);
}
} elsif ($status =~ /(.* spam.*)/i) {
$SPAM{$host}{$id}{relay} = $relay;
$SPAM{$host}{$id}{rule} = 'reject';
$SPAM{$host}{$id}{spam} = 'Spam blocked';
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{from} = &edecode($from);
$SPAM{$host}{$id}{to} = &edecode($to);
$SPAM{$host}{$id}{status} = &clear_status($status);
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'dnsbl';
$SPAMDETAIL{$host}{$id}{spam} = &clear_status($status);
}
} else {
$REJECT{$host}{$id}{relay} = $relay;
if ($reject =~ /spf/i) {
$REJECT{$host}{$id}{rule} = 'SPF rejection';
} else {
$REJECT{$host}{$id}{rule} = 'reject';
}
$REJECT{$host}{$id}{status} = &clear_status($status);
$REJECT{$host}{$id}{date} = $time_st;
$REJECT{$host}{$id}{arg1} = &edecode($from);
$REJECT{$host}{$id}{to} = $to;
}
# POSTFIX spampd reject
} elsif ($str =~ m#^([^:\s]+): reject: header X-Spam-Flag: YES from ([^;]+); from=<([^>]*)> to=<([^>]+)> [^:]+: (.*)#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
my $relay = &clean_relay(lc($2));
my $rule = 'spampd';
my $from = &edecode($3);
my $to = &edecode($4);
my $status = &clear_status($5);
return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
$SPAM{$host}{$id}{relay} = $relay;
$SPAM{$host}{$id}{rule} = 'reject';
$SPAM{$host}{$id}{spam} = $status;
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{from} = $from;
$SPAM{$host}{$id}{to} = $to;
$SPAM{$host}{$id}{status} = $status;
if (!exists $FROM{$host}{$id}{date}) {
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{from} = $from;
$FROM{$host}{$id}{size} = 0;
$FROM{$host}{$id}{nrcpts} = 1;
$FROM{$host}{$id}{relay} = $relay;
}
if (!exists $TO{$host}{$id}{date}) {
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{to}}, $to);
}
# POSTFIX milter reject message
} elsif ($str =~ m#^([^:\s]+): milter-reject: END-OF-MESSAGE from ([^\s]+) ([^;]+); from=<([^>]*)>[,]* to=<([^>]+)>#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
my $relay = &clean_relay(lc($2));
my $rule = 'milter-reject';
my $status = $3;
my $from = &edecode($4);
my $to = &edecode($5);
return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
if ($status =~ /Virus detected .*\((.*)\)/) {
$VIRUS{$host}{$id}{file} = 'Inline';
$VIRUS{$host}{$id}{virus} = $1;
$VIRUS{$host}{$id}{relay} = $relay;
$VIRUS{$host}{$id}{from} = $from;
$VIRUS{$host}{$id}{to} = $to;
$VIRUS{$host}{$id}{date} = $time_st;
if (!exists $FROM{$host}{$id}{date}) {
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{from} = $from;
$FROM{$host}{$id}{size} = 0;
$FROM{$host}{$id}{nrcpts} = 1;
$FROM{$host}{$id}{relay} = $relay;
}
if (!exists $TO{$host}{$id}{date}) {
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{to}}, $to);
$status =~ s/ \(.*//;
#push(@{$TO{$host}{$id}{status}}, &clear_status($status));
}
} elsif ($status =~ /Spam message rejected/) {
$SPAM{$host}{$id}{relay} = $relay;
$SPAM{$host}{$id}{rule} = 'reject';
$SPAM{$host}{$id}{spam} = $status;
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{from} = $from;
$SPAM{$host}{$id}{to} = $to;
$SPAM{$host}{$id}{status} = $status;
if (!exists $FROM{$host}{$id}{date}) {
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{from} = $from;
$FROM{$host}{$id}{size} = 0;
$FROM{$host}{$id}{nrcpts} = 1;
$FROM{$host}{$id}{relay} = $relay;
}
if (!exists $TO{$host}{$id}{date}) {
push(@{$TO{$host}{$id}{date}}, $time_st);
push(@{$TO{$host}{$id}{to}}, $to);
$status =~ s/ \(.*//;
#push(@{$TO{$host}{$id}{status}}, &clear_status($status));
}
} else {
$REJECT{$host}{$id}{relay} = $relay;
$REJECT{$host}{$id}{date} = $time_st;
$REJECT{$host}{$id}{arg1} = &edecode($from);
$REJECT{$host}{$id}{to} = &edecode($to);
$REJECT{$host}{$id}{rule} = $rule;
if ($status =~ /(Rejected due to SPF policy)/i) {
$REJECT{$host}{$id}{rule} = 'spf-milter';
$REJECT{$host}{$id}{status} = $1;
} elsif ($status =~ /(Rejected due to Sender-ID policy)/i) {
$REJECT{$host}{$id}{rule} = 'sid-milter';
$REJECT{$host}{$id}{status} = $1;
} else {
$REJECT{$host}{$id}{status} = &clear_status($status);
}
}
# POSTFIX some error messages
} elsif ($str =~ m#^([^:\s]+): ([^=]+)\.\.\. (.*)#) {
my $id = $KEEP_TEMPORARY{$1} || $1;
my $to = $2;
my $status = &clear_status($3);
$SYSERR{$host}{$id}{date} = $time_st;
$SYSERR{$host}{$id}{message} = $status . ': ' . $to;
# Skip lost connection after STARTTLS duplicate error
} elsif ($str =~ m#^SSL_accept error from#) {
return;
# Catch other messages with sendmail id
} elsif ($str =~ m#^([^:\s]+): (.*)#) {
my $id = $1;
my $err = $2 || '';
return if (length($id) != 14); # Skip debug lines
return if ($err =~ /clone|owner/); # Skip mailling list clone
return if ($err =~ /^(addr|Milter) /); # Skeep milter information
# Do not store if we already have it
return if (exists $SYSERR{$host}{$id} || exists $SPAM{$host}{$id} || exists $REJECT{$host}{$id} || exists $TO{$host}{$id}{status});
my $status = &clear_status($err);
return if ($status !~ /\s/); # on single word error abort
$SYSERR{$host}{$id}{date} = $time_st;
$SYSERR{$host}{$id}{message} = &clear_status($status);
# Catch SMTP AUTH
} elsif ($str =~ m#AUTH=([^,]+), relay=([^,]+), authid=([^,]+), mech=([^,]+), bits=#) {
my $authid = $3;
push(@{$AUTH{$host}{$authid}{type}}, $1);
push(@{$AUTH{$host}{$authid}{mech}}, $4);
push(@{$AUTH{$host}{$authid}{date}}, $time_st);
push(@{$AUTH{$host}{$authid}{relay}}, &clean_relay(lc($2)));
# Catch Anonymous TLS connections
} elsif ($str =~ m#Anonymous TLS connection established from ([^:]+): (.*) with cipher (.*)#) {
my $authid = 'anonymous';
push(@{$AUTH{$host}{$authid}{type}}, $2);
push(@{$AUTH{$host}{$authid}{mech}}, $3);
push(@{$AUTH{$host}{$authid}{date}}, $time_st);
push(@{$AUTH{$host}{$authid}{relay}}, &clean_relay(lc($1)));
# Catch server TLS connections
} elsif ($str =~ m#(STARTTLS=[^,]+), relay=([^,]+), version=([^,]+), (verify=[^,]+), cipher=([^,]+), bits=([^,\s]+)#) {
my $verify = $4;
push(@{$OTHER{$host}{$time_st}}, "$1, $verify");
$verify =~ /verify=(.*)/;
$STARTTLS{$host}{$time_st}{$1}++;
} elsif ($str =~ m#(STARTTLS=[^,]+), error:(.*), relay=([^,]+)#) {
push(@{$OTHER{$host}{$time_st}}, "$1, error: $2");
} else {
# Try to avoid storing some postfix debug messages
return if ($str =~ /(^[><]|^[^:]+:[^:]+:|^connection|disconnect|lookup|defer_if_permit|^idle timeout|.*attr.*|Run-time|Compiled|^process|^statistics|^warning|^(before|after) |^(match|dns|rewrite|generic|rewrite|maps|mail|ctable|smtp)_)/);
if ($str =~ /(\d{3} \d\.\d\.\d .*)/) {
$str = $1;
}
$str =~ s/.*reject=//;
push(@{$OTHER{$host}{$time_st}}, &clear_status($str));
}
}
####
# Parse MailScanner syslog output
####
sub parse_mailscanner
{
my ($date,$time,$host,$str) = @_;
return if ($str =~ /is too big for spam checks/);
my $time_st = "$date$time";
if ($str =~ /RBL checks: ([^\s]+) found in (.*)/) {
$SPAM{$host}{$1}{spam} = 'RBL checks';
$SPAM{$host}{$1}{from} = $FROM{$host}{$1}{from};
$SPAM{$host}{$1}{to} = $TO{$host}{$1}{queue_to}[0];
$SPAM{$host}{$1}{relay} = $FROM{$host}{$1}{relay};
$SPAM{$host}{$1}{date} = $time_st;
delete $TO{$host}{$1}{queue_date};
delete $TO{$host}{$1}{queue_to};
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$1}{date} = $time_st;
$SPAMDETAIL{$host}{$1}{type} = 'dnsbl';
$SPAMDETAIL{$host}{$1}{spam} = $2;
}
} elsif ($str =~ /Message ([^\s]+) from (.*) to (.*) is (?:polluriel|spam), ([^\(]+) \(([^\)]*)\)?/) {
my $id= $1;
next if ($SPAM{$host}{$id});
$SPAM{$host}{$id}{from} = $FROM{$host}{$id}{from} || $2;
$SPAM{$host}{$id}{to} = $TO{$host}{$id}{queue_to}[0] || &edecode($3);
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
$SPAM{$host}{$id}{spam} = 'SpamAssassin';
$SPAM{$host}{$id}{date} = $time_st;
my $text = $5 || '';
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'spamassassin';
$SPAMDETAIL{$host}{$id}{spam} = $text;
}
if ($SPAM{$host}{$id}{from} =~ /([a-fA-F0-9\.\:]+) \((.*)\)/) {
$SPAM{$host}{$id}{relay} = &clean_relay(lc($2));
$SPAM{$host}{$id}{from} = $1;
}
$SPAM{$host}{$id}{from} = &edecode($SPAM{$host}{$id}{from});
if ($CONFIG{SPAM_DETAIL}) {
if ($text =~ /(.*), score=(.*), requis [^,]+, (.*)/) {
$SPAMDETAIL{$host}{$id}{cache} = $1;
$SPAMDETAIL{$host}{$id}{score} = $2;
$text = $3;
if ($text =~ s/autolearn=([^,]+), //) {
$SPAMDETAIL{$host}{$id}{autolearn} = $1;
}
$SPAMDETAIL{$host}{$id}{spam} = $text;
}
}
} elsif ($str =~ /Message ([^\s]+) from (.*?) to (.*?) is (.*)/) {
my $id= $1;
next if ($SPAM{$host}{$id});
$SPAM{$host}{$id}{from} = $FROM{$host}{$id}{from} || &edecode($2);
$SPAM{$host}{$id}{to} = $TO{$host}{$id}{queue_to}[0] || &edecode($3);
delete $TO{$host}{$id}{queue_date};
delete $TO{$host}{$id}{queue_to};
$SPAM{$host}{$id}{spam} = 'RBL checks';
$SPAM{$host}{$id}{date} = $time_st;
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{type} = 'dnsbl';
$SPAMDETAIL{$host}{$id}{spam} = $4;
}
if ($SPAM{$host}{$id}{from} =~ /([a-fA-F0-9\.\:]+) \((.*)\)/) {
$SPAM{$host}{$id}{relay} = &clean_relay(lc($2));
$SPAM{$host}{$id}{from} = $1;
}
$SPAM{$host}{$id}{from} = &edecode($SPAM{$host}{$id}{from});
} elsif ($str =~ /Infected message ([^\.]+)\.[^\s]+ came from (.*)/) {
$VIRUS{$host}{$1}{from} = $2;
$VIRUS{$host}{$1}{date} = $time_st;
} elsif ($str =~ /Infected message ([^\s]+) from (.*)/) {
$VIRUS{$host}{$1}{from} = $2;
$VIRUS{$host}{$1}{date} = $time_st;
} elsif ($str =~ /Clamd::INFECTED::([^\s]+) :: \.\/([^\.]+)/) {
$VIRUS{$host}{$2}{virus} = $1;
$VIRUS{$host}{$2}{file} = 'message';
$VIRUS{$host}{$2}{date} = $time_st;
} elsif ($str =~ /\.\/([^\.]+)\.message: ([^\s]+) FOUND/) {
$VIRUS{$host}{$1}{file} = 'message';
$VIRUS{$host}{$1}{virus} = $2;
$VIRUS{$host}{$1}{date} = $time_st;
} elsif ($str =~ /Requeue: ([^\.]+)\.[^\s]+ to (.*)/) {
$KEEP_TEMPORARY{$2} = $1;
}
}
####
# Parse Amavis syslog output
####
sub parse_amavis
{
my ($date, $time ,$host,$str) = @_;
my $timest = "$date$time";
my $id = '';
if ($str =~ /\(([^\)]+)\) (Passed|Blocked) SPAM(.*?), ([^\s]+) [<]*([^\s>]*)[>]* -> ([^\s]+).*, Message-ID: [<]*([^\s>]*)[>]*,.*, Hits: ([^,]+), size: (\d+), queued_as: ([^,]+), (\d+) ms/) {
my $pid = $1;
my $status = $2;
my $relay = lc($4);
my $msgid = $7;
my $hits = $8;
my $size = $9;
$id = $10;
my $time = $11;
my $sender = &edecode($5);
my $to = &edecode($6);
if (exists $MSGID{$msgid}) {
$id = $MSGID{$msgid}{id};
delete $MSGID{$msgid};
}
$SPAM{$host}{$id}{from} = $sender;
$SPAM{$host}{$id}{to} = $to;
$SPAM{$host}{$id}{spam} = "Amavis $status Spam";
$SPAM{$host}{$id}{date} = $timest;
if (!exists $FROM{$host}{$id}{from}) {
$FROM{$host}{$id}{from} = $sender;
$FROM{$host}{$id}{date} = $timest;
$FROM{$host}{$id}{size} = $size;
$FROM{$host}{$id}{nrcpts} = 1;
}
if (!exists $FROM{$host}{$id}{relay}) {
$FROM{$host}{$id}{relay} = &clean_relay($relay);
}
if (!exists $TO{$host}{$id}{queue_to}) {
push(@{$TO{$host}{$id}{queue_date}}, $timest);
push(@{$TO{$host}{$id}{queue_to}}, $to);
}
if ($CONFIG{SPAM_DETAIL}) {
if (!exists $SPAMDETAIL{$host}{$id}) {
foreach (keys %{$SPAM{$host}{$id}}) {
$SPAMDETAIL{$host}{$id}{$_} = $SPAM{$host}{$id}{$_} if ($_ ne "spam");
}
}
$SPAMDETAIL{$host}{$id}{type} = 'amavis';
$SPAMDETAIL{$host}{$id}{score} = $hits;
$SPAMDETAIL{$host}{$id}{date} = $timest;
$SPAMDETAIL{$host}{$id}{spam} = $SPAM{$host}{$id}{spam};
}
} elsif ($str =~ /\(([^\)]+)\) (Passed|Blocked) SPAM(.*?) [<]*([^\s>]*)[>]* -> [<]*([^,>]*)[>]*,(.*) Message-ID: [<]*([^,>]+)[>]*, (.*)/) {
my $pid = $1;
my $status = $2;
my $relay = lc($3);
$id = $7;
my $queueid = $6;
my $sender = &edecode($4);
my $to = &edecode($5);
my $other = $8;
if ($queueid =~ /Queue-ID: ([^,]+)/) {
$id = $1;
} elsif ($other =~ /queued_as: ([^,]+)/) {
$id = $1;
} elsif ($str =~ /mail_id: ([^,]+)/) {
# Quarantine id
$id = $1;
}
$SPAM{$host}{$id}{from} = $sender;
$SPAM{$host}{$id}{to} = $to;
$SPAM{$host}{$id}{spam} = "Amavis $status Spam";
$SPAM{$host}{$id}{date} = $timest;
if (!exists $FROM{$host}{$id}{from}) {
$FROM{$host}{$id}{from} = $sender;
$FROM{$host}{$id}{date} = $timest;
if ($str =~ /size: (\d+)/) {
$FROM{$host}{$id}{size} = $1;
}
$FROM{$host}{$id}{nrcpts} = 1;
}
if (!exists $FROM{$host}{$id}{relay}) {
$FROM{$host}{$id}{relay} = &clean_relay($relay);
}
if (!exists $TO{$host}{$id}{queue_to}) {
push(@{$TO{$host}{$id}{queue_date}}, $timest);
push(@{$TO{$host}{$id}{queue_to}}, $to);
}
if ($CONFIG{SPAM_DETAIL}) {
if (!exists $SPAMDETAIL{$host}{$pid}) {
foreach (keys %{$SPAM{$host}{$id}}) {
$SPAMDETAIL{$host}{$pid}{$_} = $SPAM{$host}{$id}{$_} if ($_ ne "spam");
}
}
$SPAMDETAIL{$host}{$id}{type} = 'amavis';
$SPAMDETAIL{$host}{$id}{date} = $timest;
$SPAMDETAIL{$host}{$id}{spam} = $SPAM{$host}{$id}{spam};
if ($str =~ / Hits: ([\d\.]+)/) {
$SPAMDETAIL{$host}{$id}{score} = $1;
}
}
} elsif ($str =~ /\(([^\)]+)\) (Passed|Blocked) SPAM(.*?) [<]*([^\s>]*)[>]* -> [<]*([^,>]*)[>]*, Hits: ([^,]+), (.*)/) {
my $id = $1;
my $status = $2;
my $relay = lc($3);
my $sender = &edecode($4);
my $to = &edecode($5);
my $score = $6;
my $other = $7;
if (exists $AMAVIS_ID{$id}) {
$id = $AMAVIS_ID{$id};
delete $AMAVIS_ID{$id};
}
$SPAM{$host}{$id}{from} = $sender;
delete $AMAVIS_ID{$id};
$SPAM{$host}{$id}{to} = $to;
$SPAM{$host}{$id}{spam} = "Amavis $status Spam";
$SPAM{$host}{$id}{date} = $timest;
if (exists $FROM{$host}{$id}{from}) {
$SPAM{$host}{$id}{from} = $FROM{$host}{$id}{from};
$SPAM{$host}{$id}{relay} = $FROM{$host}{$id}{relay};
$SPAM{$host}{$id}{size} = $FROM{$host}{$id}{size};
}
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{type} = 'amavis';
$SPAMDETAIL{$host}{$id}{date} = $timest;
$SPAMDETAIL{$host}{$id}{spam} = $SPAM{$host}{$id}{spam};
$SPAMDETAIL{$host}{$id}{score}= $score if ($score ne '-');
}
} elsif ($str =~ /(Passed|Blocked) INFECTED \(([^\)]*)\).*, (.*?) [<]*([^\s>]*)[>]* -> [<]*([^,>]*)[>]*,(.*) Message-ID: [<]*([^,>]+)[>]*, /) {
my $virus = $2;
my $relay = lc($3);
my $from = $4;
my $to = &edecode($5);
$id = &edecode($7);
my $queue_id = $6;
if ($queue_id =~ /Queue-ID: ([^,]+),/) {
$id = $1;
}
$VIRUS{$host}{$id}{file} = 'Inline';
$VIRUS{$host}{$id}{virus} = $virus;
$VIRUS{$host}{$id}{from} = $from;
$VIRUS{$host}{$id}{to} = $to;
$VIRUS{$host}{$id}{date} = $timest;
if (!exists $FROM{$host}{$id}{from}) {
$FROM{$host}{$id}{from} = $from;
$FROM{$host}{$id}{date} = $timest;
if ($str =~ /size: (\d+)/) {
$FROM{$host}{$id}{size} = $1;
}
$FROM{$host}{$id}{nrcpts} = 1;
}
if (!exists $FROM{$host}{$id}{relay}) {
$FROM{$host}{$id}{relay} = &clean_relay($relay);
}
if (!exists $TO{$host}{$id}{queue_to}) {
push(@{$TO{$host}{$id}{queue_date}}, $timest);
push(@{$TO{$host}{$id}{queue_to}}, $to);
}
}
if ($CONFIG{SPAM_DETAIL}) {
if ($str =~ /\(([^\)]+)\) SPAM, (.*), Yes, score=([^\s]+) .* tests=(.*) autolearn=([^,]+)/) {
my $oid = $1;
my $from_to = $2;
my $score = $3;
my $spam = $4;
my $autolearn = $5;
if ($str =~ /autolearn=spam, quarantine ([^\s,]+)/) {
$id ||= $1;
}
$id ||= $oid;
$SPAMDETAIL{$host}{$id}{date} = $timest;
$SPAMDETAIL{$host}{$id}{type} = 'amavis';
$SPAMDETAIL{$host}{$id}{score} = $score;
$SPAMDETAIL{$host}{$id}{spam} = $spam;
$SPAMDETAIL{$host}{$id}{autolearn} = $autolearn;
($SPAMDETAIL{$host}{$id}{from}, $SPAMDETAIL{$host}{$id}{to}) = split(/ -> /, $from_to);
} elsif ($str =~ /\(([^\)]+)\) SPAM, (.*), Yes, score=([^\s]+).* tests=(.*)/) {
$id ||= $1;
my $from_to = $2;
$SPAMDETAIL{$host}{$id}{date} = $timest;
$SPAMDETAIL{$host}{$id}{type} = 'amavis';
$SPAMDETAIL{$host}{$id}{score} = $3;
$SPAMDETAIL{$host}{$id}{spam} = $4;
($SPAMDETAIL{$host}{$id}{from}, $SPAMDETAIL{$host}{$id}{to}) = split(/ -> /, $from_to);
} elsif ($str =~ /\(([^\)]+)\) spam_scan: score=([^\s]+) autolearn=([^\s]+) tests=(.*),/) {
$id ||= $1;
$SPAMDETAIL{$host}{$id}{date} = $timest;
$SPAMDETAIL{$host}{$id}{type} = 'amavis';
$SPAMDETAIL{$host}{$id}{score} = $2;
$SPAMDETAIL{$host}{$id}{autolearn} = $3;
$SPAMDETAIL{$host}{$id}{spam} = $4;
} elsif ($str =~ /\(([^\)]+)\) SPAM, (.*), Yes, hits=([^\s]+) .*tests=(.*), quarantine/) {
$id ||= $1;
my $from_to = $2;
$SPAMDETAIL{$host}{$id}{date} = $timest;
$SPAMDETAIL{$host}{$id}{type} = 'amavis';
$SPAMDETAIL{$host}{$id}{score} = $3;
$SPAMDETAIL{$host}{$id}{spam} = $4;
($SPAMDETAIL{$host}{$id}{from}, $SPAMDETAIL{$host}{$id}{to}) = split(/ -> /, $from_to);
}
}
}
####
# Parse clamsmtpd syslog output
####
sub parse_clamsmtpd
{
my ($date, $time ,$host,$str) = @_;
my $timest = "$date$time";
if ($str =~ /^([^:]+): from=([^,]+), to=([^,]+), status=VIRUS:(.*)/) {
my $virus = $4;
my $from = &edecode($2);
my $to = &edecode($3);
my $id = $1;
foreach my $i (keys %{$FROM{$host}}) {
if ($FROM{$host}{$i}{from} = $from) {
$id = $i;
last;
}
}
$VIRUS{$host}{$id}{file} = 'Inline';
$VIRUS{$host}{$id}{virus} = $virus;
$VIRUS{$host}{$id}{from} = $from;
$VIRUS{$host}{$id}{to} = $to;
$VIRUS{$host}{$id}{date} = $timest;
}
}
####
# Parse MimeDefang syslog output
####
sub parse_mimedefang
{
my ($date,$time,$host,$str) = @_;
my $time_st = "$date$time";
#### Store each relevant information per date and ids
#MDLOG,sendmail_queue_id,spam,score,relay,<from@sender>,<to@sdest>,subject
#MDLOG,sendmail_queue_id,virus,virus_name,relay,<from@sender>,<to@sdest>,subject
if ($str =~ /MDLOG,([^,]+),spam,([^,]+),([^,]+),([^,]+),([^,]+),(.*)/) {
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$1}{type} = 'mimedefang';
$SPAMDETAIL{$host}{$1}{score} = $2;
$SPAMDETAIL{$host}{$1}{spam} = $6;
$SPAMDETAIL{$host}{$1}{date} = $time_st;
}
$SPAM{$host}{$1}{spam} = 'mimedefang';
$SPAM{$host}{$1}{date} = $time_st;
$SPAM{$host}{$1}{relay} = &clean_relay(lc($3));
$SPAM{$host}{$1}{from} = $FROM{$host}{$1}{from} || &edecode($4);
$SPAM{$host}{$1}{to} = $TO{$host}{$1}{queue_to}[0] || &edecode($5);
delete $TO{$host}{$1}{queue_date};
delete $TO{$host}{$1}{queue_to};
} elsif ($str =~ /MDLOG,([^,]+),virus,([^,]+),([^,]+),/) {
$VIRUS{$host}{$1}{virus} = $2;
$VIRUS{$host}{$1}{file} = 'Inline';
$VIRUS{$host}{$1}{date} = $time_st;
$VIRUS{$host}{$1}{relay} = &clean_relay(lc($3));
}
}
####
# Parse Postgrey syslog output
####
sub parse_postgrey
{
my ($date,$time,$host,$str) = @_;
my $time_st = "$date$time";
if ($str =~ /action=([^,]+), reason=([^,]+), client_name=([^,]+), client_address=([^,]+), sender=([^,]+), recipient=(.*)/) {
my $action = $1;
my $status = $2;
my $relay_name = $3;
my $relay_ip = $4;
my $sender = &edecode($5);
my $to = &edecode($6);
my $id = &get_uniqueid();
$status =~ s/ \(.*//;
$POSTGREY{$host}{$id}{action} = $action;
$POSTGREY{$host}{$id}{relay} = &clean_relay($relay_name || $relay_ip);
$POSTGREY{$host}{$id}{from} = &edecode($sender);
$POSTGREY{$host}{$id}{to} = &edecode($to);
$POSTGREY{$host}{$id}{status} = $status;
$POSTGREY{$host}{$id}{date} = $time_st;
} elsif ($str =~ s/^grey:\s+//) {
&parse_sqlgrey($date,$time_st,$host,$str);
} elsif ($str =~ s/^spam:\s+//) {
&parse_sqlgrey_spam($date,$time_st,$host,$str);
}
}
####
# Parse sqlgrey spam syslog output
####
sub parse_sqlgrey_spam
{
my ($date,$time_st,$host,$str) = @_;
if ($str =~ /^(.*): (.*) -> ([^\s]+) /) {
my $relay = lc($1);
my $sender = &edecode($2);
my $to = &edecode($3);
my $id = &get_uniqueid();
$SPAM{$host}{$id}{relay} = $relay;
$SPAM{$host}{$id}{from} = $sender;
$SPAM{$host}{$id}{to} = $to;
$SPAM{$host}{$id}{spam} = "sqlgrey spam";
$SPAM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{from} = $sender;
$FROM{$host}{$id}{date} = $time_st;
$FROM{$host}{$id}{nrcpts} = 1;
$FROM{$host}{$id}{relay} = $relay;
}
}
####
# Parse sqlgrey syslog output
####
sub parse_sqlgrey
{
my ($date,$time_st,$host,$str) = @_;
my $id = &get_uniqueid();
if ($str =~ /^domain awl match: updating ([^,]+), (.*)/) {
$POSTGREY{$host}{$id}{status} = 'passed domain awl match';
$POSTGREY{$host}{$id}{action} = 'passed';
$POSTGREY{$host}{$id}{relay} = &clean_relay($1);
$POSTGREY{$host}{$id}{from} = 'unset@'. &edecode($2);
$POSTGREY{$host}{$id}{date} = $time_st;
} elsif ($str =~ /^from awl match: updating ([^,]+), ([^\(]+)/) {
$POSTGREY{$host}{$id}{status} = 'passed from awl match';
$POSTGREY{$host}{$id}{action} = 'passed';
$POSTGREY{$host}{$id}{relay} = &clean_relay($1);
$POSTGREY{$host}{$id}{from} = &edecode($2);
$POSTGREY{$host}{$id}{date} = $time_st;
} elsif ($str =~ /^early reconnect: ([^,]+), (.*) -> (.*)/) {
$POSTGREY{$host}{$id}{status} = 'early reconnect';
$POSTGREY{$host}{$id}{action} = 'early';
$POSTGREY{$host}{$id}{relay} = &clean_relay($1);
$POSTGREY{$host}{$id}{from} = &edecode($2);
$POSTGREY{$host}{$id}{to} = &edecode($3);
$POSTGREY{$host}{$id}{date} = $time_st;
} elsif ($str =~ /^reconnect ok: ([^,]+), (.*) -> (.*) \((.*)\)/) {
$POSTGREY{$host}{$id}{status} = 'passed reconnect ok';
$POSTGREY{$host}{$id}{action} = 'passed';
$POSTGREY{$host}{$id}{relay} = &clean_relay($1);
$POSTGREY{$host}{$id}{from} = &edecode($2);
$POSTGREY{$host}{$id}{to} = &edecode($3);
$POSTGREY{$host}{$id}{date} = $time_st;
} elsif ($str =~ /^new: ([^,]+), (.*) -> (.*)/) {
$POSTGREY{$host}{$id}{status} = 'new delayed';
$POSTGREY{$host}{$id}{action} = 'delayed';
$POSTGREY{$host}{$id}{relay} = &clean_relay($1);
$POSTGREY{$host}{$id}{from} = &edecode($2);
$POSTGREY{$host}{$id}{to} = &edecode($3);
$POSTGREY{$host}{$id}{date} = $time_st;
}
}
####
# Parse spamd syslog output
####
sub parse_spamd
{
my ($date,$time,$host,$str) = @_;
my $time_st = "$date$time";
#### Store each relevant information per date and ids
if ($str =~ /result: Y ([^\s]+) - (.*) scantime=.*mid=<(.*)>,.*autolearn=(.*)/) {
my $score = $1;
my $spam = $2;
my $msgid = $3;
my $autolearn = $4;
my $id = &get_uniqueid();
$SPAM{$host}{$id}{spam} = 'spamd';
$SPAM{$host}{$id}{date} = $time_st;
$SPAM{$host}{$id}{mid} = $msgid;
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$id}{type} = 'spamdmilter';
$SPAMDETAIL{$host}{$id}{score} = $score;
$SPAMDETAIL{$host}{$id}{spam} = $spam;
$SPAMDETAIL{$host}{$id}{date} = $time_st;
$SPAMDETAIL{$host}{$id}{mid} = $msgid;
$SPAMDETAIL{$host}{$id}{autolearn} = $autolearn;
}
foreach my $mid (keys %{$FROM{$host}}) {
next if (!exists $FROM{$host}{$mid}{msgid});
# Some message id can be truncated in from log and full in spamd message
if ($SPAM{$host}{$id}{mid} =~ /^\Q$FROM{$host}{$mid}{msgid}\E/) {
$SPAM{$host}{$mid}{from} = $FROM{$host}{$mid}{sender};
$SPAM{$host}{$mid}{spam} = $SPAM{$host}{$id}{spam};
$SPAM{$host}{$mid}{date} = $SPAM{$host}{$id}{date};
$SPAM{$host}{$mid}{mid} = $SPAM{$host}{$id}{mid};
$FROM{$host}{$mid}{msgid} = $SPAM{$host}{$id}{mid};
delete $SPAM{$host}{$id};
if ($CONFIG{SPAM_DETAIL}) {
$SPAMDETAIL{$host}{$mid}{type} = 'spamdmilter';
$SPAMDETAIL{$host}{$mid}{score} = $SPAMDETAIL{$host}{$id}{score};
$SPAMDETAIL{$host}{$mid}{spam} = $SPAMDETAIL{$host}{$id}{spam};
$SPAMDETAIL{$host}{$mid}{date} = $SPAMDETAIL{$host}{$id}{date};
$SPAMDETAIL{$host}{$mid}{mid} = $mid;
$SPAMDETAIL{$host}{$mid}{autolearn} = $SPAMDETAIL{$host}{$id}{autolearn};
delete $SPAMDETAIL{$host}{$id};
}
last;
}
}
}
}
####
# Parse DKIM/SPF syslog output
####
sub parse_spf_dkim
{
my ($date,$time,$host,$str) = @_;
my $time_st = "$date$time";
#Jun 7 06:22:58 server opendmarc[980]: 2693E3640260: news.vendita--flash.com none
if ($str =~ /^([^:]+): SPF\(([^\)]+)\): ([^\s]+) ([^\s]+)$/) {
my $id = $1;
$SPF_DKIM{$host}{spf}{$id}{rule} = $2;
$SPF_DKIM{$host}{spf}{$id}{status} = $4;
$SPF_DKIM{$host}{spf}{$id}{domain} = &clean_relay($3);
$SPF_DKIM{$host}{spf}{$id}{date} = $time_st;
#Jun 7 06:22:58 server opendkim[953]: 2693E3640260: DKIM verification successful
} elsif ($str =~ /^([^:]+): DKIM (.*)$/) {
my $id = $1;
$SPF_DKIM{$host}{dkim}{$id}{status} = $2;
$SPF_DKIM{$host}{dkim}{$id}{date} = $time_st;
#Jun 7 08:43:51 server opendkim[953]: 07EE63640A96: DKIM-Signature field added (s=default, d=mydomain.com)
} elsif ($str =~ /^([^:]+): DKIM-([^\(]+) \(s=([^,]+), d=([^\)]+)\)$/) {
my $id = $1;
$SPF_DKIM{$host}{dkim}{$id}{status} = $2;
$SPF_DKIM{$host}{dkim}{$id}{rule} = $3;
$SPF_DKIM{$host}{dkim}{$id}{domain} = $4;
$SPF_DKIM{$host}{dkim}{$id}{date} = $time_st;
}
}
####
# Decode email address and keep only email part
####
sub edecode
{
my ($addr) = @_;
if ($addr =~ /=\?[^\?]+\?(.)\?(.*)?=/s) {
if (uc($1) eq 'B') {
$addr = decode_base64($1);
} elsif (uc($1) eq 'Q') {
$addr = decode_qp($1);
}
}
$addr =~ s#^\s+##;
$addr =~ s#\s+$##;
$addr =~ s#[<>]##g;
$addr =~ s#,$##;
$addr =~ s#:##g;
$addr =~ s#'##g;
$addr =~ s# \(\d+/\d+\)##g;
if ($addr !~ /\@/) {
$addr .= $CONFIG{DEFAULT_DOMAIN} || '@localhost';
}
return lc($addr);
}
####
# Decode subject
####
sub decode_subject
{
my ($str) = @_;
while ($str =~ /=\?[^\?]+\?(.)\?(.*?)\?=/) {
my $subject = $1;
if (uc($subject) eq 'B') {
$subject = decode_base64($2);
} elsif (uc($subject) eq 'Q') {
$subject = decode_qp($2);
}
$str =~ s/=\?[^\?]+\?(.)\?(.*?)\?=/$subject/;
}
while ($str =~ /=\?[^\?]+\?(.)\?(.*?)/) {
my $subject = $1;
if (uc($subject) eq 'B') {
$subject = decode_base64($2);
} elsif (uc($subject) eq 'Q') {
$subject = decode_qp($2);
}
$str =~ s/=\?[^\?]+\?(.)\?(.*?)/$subject/;
}
$str =~ s/\?\?//g;
return $str;
}
####
# Clean relay address
####
sub clean_relay
{
my ($relay) = @_;
if ($relay =~ m#\b([a-fA-F0-9\.\:]+) \(may be forged#) {
$relay = $1;
} elsif ($relay =~ m#localhost|127\.0\.0\.1#) {
return 'localhost';
} elsif ( $relay =~ s/\[([^\]]+)\]// ) {
my $fqdn = $relay;
my $ip = $1;
$fqdn =~ s#:.*##;
if (!$fqdn || ($fqdn eq 'unknown')) {
$relay = $ip;
} elsif ($fqdn =~ /[\s,]/) {
$relay = $ip;
} else {
$relay = $fqdn;
}
} elsif ( $relay =~ s/\(([a-fA-F0-9\.\:]+)\)// ) {
$relay = $1;
}
$relay =~ s#^\s+##;
$relay =~ s#\s+.*##;
$relay =~ s#\.$##;
$relay =~ s#\s.*##;
$relay =~ s/:/_/g; # fix ipv6 to remove data field separator
return $relay;
}
####
# Set script internal date/time format from localtime call
# Format: YYYYMMDDHHMMSS
####
sub format_time
{
my ($sec,$min,$hour,$mday,$mon,$year) = @_;
$hour = sprintf("%02d", $hour);
$min = sprintf("%02d", $min);
$sec = sprintf("%02d", $sec);
return 1900+$year . sprintf("%02d", $mon+1) . sprintf("%02d", $mday) . "$hour$min$sec";
}
####
# Flush memory stored data to disk
####
sub flush_data
{
my $final = shift;
# In incremental mode if there's still no line parsed get out of there
return if ($OLD_LAST_PARSED ne '');
####
# Data are saved on disk as follow:
# $host/$year/$month/$day/filename.dat
####
# Init greylisting temporary storage
%GREYLIST = ();
my %EXCLUDED = ();
# Save senders informations first
&dprint("Writing sender data to disk...");
my $nobj = 0;
foreach my $host (keys %FROM) {
my %senders = ();
foreach my $id (keys %{$FROM{$host}}) {
if (exists $POSTFIX_PLUGIN_TEMP_DELIVERY{$id}) {
delete $FROM{$host}{$id};
next;
}
# Check from sender or sender relay exclusion
if ($CONFIG{EXCLUDE_FROM} && ($FROM{$host}{$id}{from} =~ /^$CONFIG{EXCLUDE_FROM}$/)) {
$EXCLUDED{$id} = 1;
delete $FROM{$host}{$id};
next;
}
if ($CONFIG{EXCLUDE_RELAY} && ($FROM{$host}{$id}{relay} =~ /^$CONFIG{EXCLUDE_RELAY}$/)) {
$EXCLUDED{$id} = 1;
delete $FROM{$host}{$id};
next;
}
foreach (keys %MSGID) {
delete $MSGID{$_} if ($MSGID{$_}{id} eq $id);
}
delete $SKIPMSG{$id};
next if ($FROM{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
# Key: year/month/day, Format: Hour:Id:Sender:Size:Nrcpts:Relay:Subject
$senders{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $FROM{$host}{$id}{from} . ':' . $FROM{$host}{$id}{size} . ':' . $FROM{$host}{$id}{nrcpts} . ':' . $FROM{$host}{$id}{relay} . ':' . $FROM{$host}{$id}{subject} ."\n";
$nobj++;
# Complete Spam information
if (exists $SPAM{$host}{$id}) {
$SPAM{$host}{$id}{date} = $FROM{$host}{$id}{date};
$SPAM{$host}{$id}{from} = $FROM{$host}{$id}{from};
}
# Find real sendmail id for Amavis virus message
my $msgid = $FROM{$host}{$id}{msgid} || '';
if ($msgid && exists($VIRUS{$host}{$msgid})) {
foreach my $k (keys %{$VIRUS{$host}{$msgid}}) {
$VIRUS{$host}{$id}{$k} = $VIRUS{$host}{$msgid}{$k};
}
delete $VIRUS{$host}{$msgid};
}
# Find real sendmail id for Amavis spam message
if ($msgid && exists($SPAM{$host}{$msgid})) {
foreach my $k (keys %{$SPAM{$host}{$msgid}}) {
$SPAM{$host}{$id}{$k} = $SPAM{$host}{$msgid}{$k};
}
delete $SPAM{$host}{$msgid};
}
if ($msgid && exists($SPAMDETAIL{$host}{$msgid})) {
foreach my $k (keys %{$SPAMDETAIL{$host}{$msgid}}) {
$SPAMDETAIL{$host}{$id}{$k} = $SPAMDETAIL{$host}{$msgid}{$k};
}
delete $SPAMDETAIL{$host}{$msgid};
}
}
delete $FROM{$host};
foreach my $dir (keys %senders) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/senders.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/senders.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $senders{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj sender objects");
# clear all senders memory storage
%FROM = ();
&dprint("Writing reject data to disk...");
$nobj = 0;
# Save rejected messages
foreach my $host (keys %REJECT) {
my %rejected = ();
foreach my $id (keys %{$REJECT{$host}}) {
if (exists $EXCLUDED{$id}) {
delete $REJECT{$host}{$id};
next;
}
$REJECT{$host}{$id}{date} =~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/;
# Key: year/month/day, Format: Hour:Id:Rule:Relay:Arg1:Status
$rejected{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $REJECT{$host}{$id}{rule} . ':' . $REJECT{$host}{$id}{relay} . ':' . $REJECT{$host}{$id}{arg1} . ':' . $REJECT{$host}{$id}{status} . "\n";
$nobj++;
}
delete $REJECT{$host};
foreach my $dir (keys %rejected) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/rejected.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/rejected.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $rejected{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj reject object.");
# clear all senders memory storage
%REJECT = ();
&dprint("Writing DSN data to disk...");
$nobj = 0;
# Save DSN messages
foreach my $host (keys %DSN) {
my %dsned = ();
foreach my $id (keys %{$DSN{$host}}) {
if (exists $EXCLUDED{$id}) {
delete $DSN{$host}{$id};
next;
}
next if ($DSN{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
# Key: year/month/day, Format: Hour:Id:SourceId:Status
$dsned{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $DSN{$host}{$id}{srcid} . ':' . $DSN{$host}{$id}{status} . "\n";
$nobj++;
}
delete $DSN{$host};
foreach my $dir (keys %dsned) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/dsn.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/dsn.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $dsned{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj DSN object.");
# clear all senders memory storage
%DSN = ();
# Save recipients informations
&dprint("Writing recipient data to disk...");
$nobj = 0;
foreach my $host (keys %TO) {
my %rcpts = ();
foreach my $id (keys %{$TO{$host}}) {
if (exists $POSTFIX_PLUGIN_TEMP_DELIVERY{$id}) {
delete $TO{$host}{$id};
delete $POSTFIX_PLUGIN_TEMP_DELIVERY{$id};
next;
}
if (exists $EXCLUDED{$id}) {
delete $TO{$host}{$id};
next;
}
for (my $i = 0; $i <= $#{$TO{$host}{$id}{date}}; $i++) {
# Key: year/month/day, Format: Hour:Id:Rcpt:Relay:Status
next if ($TO{$host}{$id}{date}[$i] !~ /(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
$rcpts{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $TO{$host}{$id}{to}[$i] . ':' . $TO{$host}{$id}{relay}[$i] . ':' . $TO{$host}{$id}{status}[$i] . "\n";
$nobj++;
}
for (my $i = 0; $i <= $#{$TO{$host}{$id}{queue_date}}; $i++) {
# Key: year/month/day, Format: Hour:Id:Rcpt:Relay:Status
next if ($TO{$host}{$id}{queue_date}[$i] !~ /(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
$rcpts{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $TO{$host}{$id}{queue_to}[$i] . ":none:Queued\n";
$nobj++;
}
# Complete Spam information
if (exists $SPAM{$host}{$id} && !exists $SPAM{$host}{$id}{to}) {
$SPAM{$host}{$id}{to} = $TO{$host}{$id}{to}[0];
}
}
delete $TO{$host};
foreach my $dir (keys %rcpts) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/recipient.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/recipient.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $rcpts{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj recipient object.");
# clear all recipients memory storage
%TO = ();
# Save Spam objects
&dprint("Writing Spam data to disk...");
$nobj = 0;
foreach my $host (keys %SPAM) {
my %spams = ();
foreach my $id (keys %{$SPAM{$host}}) {
if (exists $EXCLUDED{$id}) {
delete $SPAM{$host}{$id};
next;
}
next if ($SPAM{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
# Key: year/month/day, Format: Hour:Id:from:to:spam
$spams{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $SPAM{$host}{$id}{from} . ':' . $SPAM{$host}{$id}{to} . ':' . $SPAM{$host}{$id}{spam} . "\n";
$nobj++;
}
delete $SPAM{$host};
foreach my $dir (keys %spams) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/spam.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/spam.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $spams{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj spam object.");
# clear all spams memory storage
%SPAM = ();
%SPAMPD = ();
# Save Spam detail objects
&dprint("Writing Spam detail data to disk...");
$nobj = 0;
foreach my $host (keys %SPAMDETAIL) {
my %spamdetails = ();
foreach my $id (keys %{$SPAMDETAIL{$host}}) {
if (exists $EXCLUDED{$id}) {
delete $SPAMDETAIL{$host}{$id};
next;
}
next if (!$SPAMDETAIL{$host}{$id}{type});
next if ($SPAMDETAIL{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
# Key: year/month/day, Format: Hour:Id:type:score:cache:autolearn:spam
$spamdetails{"$1/$2/$3"}{$SPAMDETAIL{$host}{$id}{type}} .= "$4$5$6" . ':' . $id . ':' . $SPAMDETAIL{$host}{$id}{type} . ':' . $SPAMDETAIL{$host}{$id}{score} . ':' . $SPAMDETAIL{$host}{$id}{cache} . ':' . $SPAMDETAIL{$host}{$id}{autolearn} . ':' . $SPAMDETAIL{$host}{$id}{spam} . "\n";
$nobj++;
}
delete $SPAMDETAIL{$host};
foreach my $dir (keys %spamdetails) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
foreach my $type (keys %{$spamdetails{$dir}}) {
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/$type.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/$type.dat: $!");
&logerror("Data will be lost.");
next;
} else {
&dprint("Writing Spam detail for $type into $CONFIG{OUT_DIR}/$host/$dir/$type.dat");
print OUT $spamdetails{$dir}{$type};
close(OUT);
}
}
}
}
&dprint("\tWrote $nobj spam detail object.");
# clear all spams memory storage
%SPAMDETAIL = ();
# Save Postgrey objects
&dprint("Writing Postgrey detail data to disk...");
$nobj = 0;
foreach my $host (keys %POSTGREY) {
my %postgreys = ();
foreach my $id (keys %{$POSTGREY{$host}}) {
if (exists $EXCLUDED{$id}) {
delete $POSTGREY{$host}{$id};
next;
}
next if ($POSTGREY{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
# Key: year/month/day, Format: Hour:Id:relay:from:to:action:status
$postgreys{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $POSTGREY{$host}{$id}{relay} . ':' . $POSTGREY{$host}{$id}{from} . ':' . $POSTGREY{$host}{$id}{to} . ':' . $POSTGREY{$host}{$id}{action} . ':' . $POSTGREY{$host}{$id}{status} . "\n";
$nobj++;
}
delete $POSTGREY{$host};
foreach my $dir (keys %postgreys) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/postgrey.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/postgrey.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $postgreys{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj postgrey object.");
# clear all postgrey memory storage
%POSTGREY = ();
# SaveSPF/DKIM objects
&dprint("Writing SPF/DKIM detail data to disk...");
$nobj = 0;
foreach my $host (keys %SPF_DKIM) {
my %spf_dkims = ();
foreach my $kind (keys %{$SPF_DKIM{$host}}) {
foreach my $id (keys %{$SPF_DKIM{$host}{$kind}}) {
if (exists $EXCLUDED{$id}) {
delete $SPF_DKIM{$host}{$kind}{$id};
next;
}
next if ($SPF_DKIM{$host}{$kind}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
# Key: year/month/day, Format: Hour:Id:type:rule:domain:status
$spf_dkims{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $kind . ':' . $SPF_DKIM{$host}{$kind}{$id}{rule} . ':' . $SPF_DKIM{$host}{$kind}{$id}{domain} . ':' . $SPF_DKIM{$host}{$kind}{$id}{status} . "\n";
$nobj++;
}
}
foreach my $dir (keys %spf_dkims) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/spf_dkim.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/spf_dkim.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $spf_dkims{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj spf/dkim object.");
# clear all spf/dkim memory storage
%SPF_DKIM = ();
# Save Virus objects
&dprint("Writing Virus data to disk...");
$nobj = 0;
foreach my $host (keys %VIRUS) {
my %viruses = ();
foreach my $id (keys %{$VIRUS{$host}}) {
if (exists $EXCLUDED{$id}) {
delete $VIRUS{$host}{$id};
next;
}
next if ($VIRUS{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
# Key: year/month/day, Format: Hour:Id:file:virus
$viruses{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $VIRUS{$host}{$id}{file} . ':' . $VIRUS{$host}{$id}{virus} . "\n";
$nobj++;
}
delete $VIRUS{$host};
foreach my $dir (keys %viruses) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/virus.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/virus.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $viruses{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj virus object.");
# clear all viruses memory storage
%VIRUS = ();
# Save syserr objects
&dprint("Writing syserr data to disk...");
$nobj = 0;
foreach my $host (keys %SYSERR) {
my %errors = ();
foreach my $id (keys %{$SYSERR{$host}}) {
if (exists $EXCLUDED{$id}) {
delete $SYSERR{$host}{$id};
next;
}
next if ($SYSERR{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
# Key: year/month/day, Format: Hour:Id:Error
$errors{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $SYSERR{$host}{$id}{message} . "\n";
$nobj++;
}
delete $SYSERR{$host};
foreach my $dir (keys %errors) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/syserr.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/syserr.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $errors{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj syserr object.");
# clear all syserr memory storage
%SYSERR = ();
# Save other message objects
&dprint("Writing warning message data to disk...");
$nobj = 0;
foreach my $host (keys %OTHER) {
my %errors = ();
foreach my $dt (keys %{$OTHER{$host}}) {
next if ($dt !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
foreach my $v (@{$OTHER{$host}{$dt}}) {
# Key: year/month/day, Format: Hour:Error
$errors{"$1/$2/$3"} .= "$4$5$6" . ':' . $v . "\n";
$nobj++;
}
}
delete $OTHER{$host};
foreach my $dir (keys %errors) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/other.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/other.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $errors{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj warning message object.");
# clear all warning message memory storage
%OTHER = ();
# Save STARTTLS stats
&dprint("Writing STARTTLS statistics to disk...");
$nobj = 0;
foreach my $host (keys %STARTTLS) {
my %starttls = ();
foreach my $dt (keys %{$STARTTLS{$host}}) {
next if ($dt !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
# Key: year/month/day, Format: Hour:Id:FAIL=count;NO=count;OK=count;
$starttls{"$1/$2/$3"} .= "$4$5$6" . ':';
foreach my $v (keys %{$STARTTLS{$host}{$dt}}) {
$starttls{"$1/$2/$3"} .= $v . '=' . ($STARTTLS{$host}{$dt}{$v} || 0) . ';';
$nobj++;
}
$starttls{"$1/$2/$3"} .= "\n";
$starttls{"$1/$2/$3"} =~ s/;$//s;
}
delete $STARTTLS{$host};
foreach my $dir (keys %starttls) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/starttls.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/starttls.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $starttls{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj STARTTLS objects.");
# clear all warning message memory storage
%STARTTLS = ();
# Save auth message objects
&dprint("Writing warning auth data to disk...");
$nobj = 0;
foreach my $host (keys %AUTH) {
my %authent = ();
foreach my $id (keys %{$AUTH{$host}}) {
if (exists $EXCLUDED{$id}) {
delete $AUTH{$host}{$id};
next;
}
for (my $i = 0; $i <= $#{$AUTH{$host}{$id}{date}}; $i++) {
next if ($AUTH{$host}{$id}{date}[$i] !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
# Key: year/month/day, Format: date:id:relay:mech:type
$authent{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $AUTH{$host}{$id}{relay}[$i] . ':' . $AUTH{$host}{$id}{mech}[$i] . ':' . $AUTH{$host}{$id}{type}[$i] . "\n";
$nobj++;
}
}
delete $AUTH{$host};
foreach my $dir (keys %authent) {
if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
&create_directory("$host/$dir");
}
if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/auth.dat") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/auth.dat: $!");
&logerror("Data will be lost.");
next;
} else {
print OUT $authent{$dir};
close(OUT);
}
}
}
&dprint("\tWrote $nobj auth object.");
# clear all auth storage
%AUTH = ();
# Write last parsed data
if (!$CONFIG{FORCE}) {
if (not open(OUT, ">$CONFIG{OUT_DIR}/$LAST_PARSE_FILE") ) {
&logerror("Can't write to file $CONFIG{OUT_DIR}/$LAST_PARSE_FILE: $!");
} else {
&dprint("Writing last parsed line and last log file reading offset...");
print OUT "$LAST_PARSED\t$OLD_OFFSET";
close(OUT);
}
}
}
####
# Create output directory tree
####
sub create_directory
{
my $dest = shift;
my $curdir = '';
foreach my $d (split(/\//, $dest)) {
$curdir .= $d . '/';
if (!-d "$CONFIG{OUT_DIR}/$curdir") {
if (not mkdir("$CONFIG{OUT_DIR}/$curdir")) {
&logerror("Can't create directory $CONFIG{OUT_DIR}/$curdir: $!");
&logerror("Data will be lost.");
return 0;
}
}
}
return 1;
}
####
# Routine used to log sendmailanalyzer errors or send emails alert if requested
####
sub logerror
{
my $str = shift;
print STDERR "ERROR: $str\n";
}
####
# Read configuration file
####
sub read_config
{
my $file = shift;
if (!-e $file) {
$file = '/etc/sendmailanalyzer.conf';
}
if (!-e $file) {
&logerror("Configuration file $file doesn't exists");
return;
} else {
if (not open(IN, $file)) {
&logerror("Can't read configuration file $file: $!");
} else {
while (<IN>) {
chomp;
s/#.*//;
s/^[\s\t]+//;
s/[\s\t]$//;
if ($_ ne '') {
my ($var, $val) = split(/[\s\t]+/, $_, 2);
$CONFIG{$var} = $val if (!defined $CONFIG{$var} && ($val ne ''));
}
}
close(IN);
}
}
# Set default values
$CONFIG{LOG_FILE} ||= '/var/log/maillog';
$CONFIG{ZCAT_PROG} ||= '/usr/bin/zcat';
$CONFIG{TAIL_PROG} ||= '/usr/bin/tail';
$CONFIG{TAIL_ARGS} ||= '-n 0 -f';
$CONFIG{OUT_DIR} ||= '/var/www/sendmailanalyzer';
$CONFIG{PID_DIR} ||= $CONFIG{PID_FILE};
$CONFIG{DEBUG} ||= 0;
$CONFIG{FULL} ||= 0;
$CONFIG{FORCE} ||= 0;
$CONFIG{BREAK} ||= 0;
$CONFIG{DELAY} ||= 5;
$CONFIG{MTA_NAME} ||= 'sm-mta|sendmail|postfix|spampd';
$CONFIG{MAILSCAN_NAME} ||= 'MailScanner';
$CONFIG{CLAMD_NAME} ||= 'clamd';
$CONFIG{MD_NAME} ||= 'mimedefang.pl';
$CONFIG{AMAVIS_NAME} ||= 'amavis';
if (!exists $CONFIG{SPAM_DETAIL}) {
$CONFIG{SPAM_DETAIL} = 1;
}
$CONFIG{CLAMSMTPD_NAME} = 'clamsmtpd';
$CONFIG{MTA_NAME} =~ s/[\s\t,;]+/\|/g;
$CONFIG{PID_DIR} ||= '/var/run';
$CONFIG{POSTGREY_NAME} ||= 'postgrey|sqlgrey';
$CONFIG{SPAMD_NAME} ||= 'spamd';
$CONFIG{SPF_DKIM_NAME} ||= 'opendmarc|opendkim';
}
####
# Routine to remove any spercific part of status line to report
# generic status
####
sub clear_status
{
my ($status, $id) = @_;
# Try to avoid doublon when postfix send the message to a plugin
# and then the plugin will use postfix again to send the message.
if ($status =~ /sent \(250 .* ([0-9A-F]+)\)/) {
$POSTFIX_PLUGIN_TEMP_DELIVERY{$1} = $id;
}
# Try to detect Amavis id from messages status
if ($status =~ /^[^\s]+ \(.*, id=(\d+\-\d+) - (.*)\)/) {
$AMAVIS_ID{$1} = $id;
}
# Anonymize ip addresse in status message
$status =~ s/\[[a-fA-F0-9\.\:]+\]/.../g;
$status =~ s/(IP name possibly forged).*/$1/;
$status =~ s/(IP name lookup failed).*/$1/;
if ($status =~ /^Sent.*/i) {
return 'Sent';
} elsif ($status =~ /Deferred.*/i) {
return 'Deferred';
} elsif ($status =~ s/collect: //) {
$status =~ s/ from.*//;
return $status;
} elsif ($status =~ /bounced.*/i) {
return 'Bounced';
} elsif ($status =~ /hostname (.*) does not resolve to address ([^:]+): (.*)/) {
return "hostname does not resolve to address, $3";
} elsif ($status =~ /hostname (.*) does not resolve to address ([^:]+)/) {
return "hostname does not resolve to address";
} elsif ($status =~ /(numeric domain name)/) {
return "numeric domain name";
} elsif ($status =~ /(address not listed for hostname)/) {
return $1;
} elsif ($status =~ /warning: (Illegal address syntax).* in ([^\s]+ command):/) {
return "$1 $2";
} elsif ($status =~ /warning: (non-SMTP command)/) {
return "$1";
} elsif ($status =~ /warning: [^:]+:\d+ ([^:]+)/) {
return $1;
} elsif ($status =~ /warning: [^:]+: ([^:]+)/) {
return $1;
} elsif ($status =~ /did not issue/) {
$status =~ s/.*did not issue/No/;
return $status;
} elsif ($status =~ /(Greylisting in action)/i) {
return $1;
} elsif ($status =~ /(lost input channel ).*(after.*)/) {
$status = $1 . $2;
return $status;
} elsif ($status =~ /(timeout waiting for input ).*(during.*)/) {
$status = $1 . $2;
return $status;
} elsif ($status =~ /(timeout writing message).*(: [^:]+?)( by |$)/) {
$status = $1 . $2;
return $status;
} elsif ($status =~ /(timeout writing message)/) {
return $1;
} elsif ($status =~ /(rejecting commands ).* (due to pre-greeting traffic)/) {
return $1 . $2;
} elsif ($status =~ /(rejecting commands ).*(due to.*)/) {
$status = $1 . $2;
return $status;
} elsif ($status =~ /(readqf: ).*:( .*)/) {
$status = $1 . $2;
return $status;
} elsif ($status =~ /(Syntax error in mailbox address)/) {
return $1;
} elsif ($status =~ /(Possible SMTP RCPT flood, throttling)/) {
return $1;
} elsif ($status =~ /(Domain name required for sender address)/) {
return $1;
} elsif ($status =~ /(STARTTLS=[^,]+),.*(, verify=[^,]+)/) {
return $1 . $2;
} elsif ($status =~ /, stat=(.*)/) {
return $1;
} elsif ($status =~ /(rejecting connections on daemon[^:]*: [^:]*)/) {
return $1;
} elsif ($status =~ /(VRFY ([^\s]+) rejected)/) {
return "VRFY rejected";
} elsif ($status =~ /: (sender notify:.*)/) {
return $1;
} elsif ($status =~ /(config error: mail loops back to me)/) {
return $1;
} elsif ($status =~ /(Authentication-Warning:).*[a-fA-F0-9\.\:]+(.*)/) {
return "$1 $2";
} elsif ($status =~ /(Authentication-Warning: [^:]+: [^\s]+ set sender) to .*(using -f)/) {
return "$1 $2";
} elsif ($status =~ /Losing .* savemail panic/) {
return "Losing message, savemail panic";
} elsif ($status =~ /(probable open proxy)/) {
return $1;
} elsif ($status =~ /(Too many hops)/) {
return $1;
} elsif ($status =~ /^(.*come back) in/) {
return "$1 later";
} elsif ($status =~ /^(setsender: [^:]+):/) {
my $ret = $1;
$ret =~ s/ SIZE=.*//;
return $ret . " attack attempt";
} elsif ($status =~ /(User unknown)/) {
return $1;
} elsif ($status =~ /: ([^:]+): Please see .*/) {
return $1;
} elsif ($status =~ /(Greylisted), see http.*/) {
return $1;
} elsif ($status =~ /(improper command pipelining after RCPT).*/) {
return $1;
} elsif ($status =~ /(can't identify domain) in.*/) {
return $1;
} elsif ($status =~ /.*(Illegal address syntax).*( in .* command)/) {
return "$1$2";
} elsif ($status =~ /(improper command pipelining after HELO)/i) {
return $1;
} elsif ($status =~ /(You are still greylisted)/i) {
return $1;
} elsif ($status =~ /(.*): (possible SMTP attack): (.*)/i) {
return "$2 from $1 ($3)";
} elsif ($status =~ /(Domain of sender address) ([^\s]+) (.*)/i) {
return "$1 $3: $2";
} elsif ($status =~ /\d{3} \d\.\d\.\d <[^>]+>[:\s\.]*(.*)/) {
return $1;
} elsif ($status =~ /\d{3} \d\.\d\.\d [^@]+\@[^\s]+ (.*)/) {
return $1;
} elsif ($status =~ /^\d{3} \d\.\d\.\d (.*?) (has exceeded .*)/) {
return $2;
} elsif ($status =~ /^\d{3} \d\.\d\.\d ([^\.]+)/) {
my $str = $1;
$str =~ s/://;
return $str;
} elsif ($status =~ /^\d\.\d\.\d (.*)/) {
return $1;
}
$status =~ s/\.\.\..*//;
$status =~ s/ with .*//;
$status =~ s/ by .*//;
$status =~ s/ \(.*//;
$status =~ s/ '.'//g;
$status =~ s/\.$//;
$status =~ s/;.*$//;
if ($status =~ /(\d{3} \d\.\d\.\d).*[^a-z\s:\.]+.*/i) {
return $1;
}
return $status;
}
####
# Routine to remove any spercific part of rejected status line to report
# generic status
####
sub clear_rejection_status
{
my $status = shift;
if ($status =~ /(\d{3} \d\.\d\.\d)/) {
return $1;
}
$status =~ s/^.*reject=//;
$status =~ s/^.*\.\.\.\s*//;
$status =~ s/[^\d]\..*$//;
return $status;
}
####
# Check wether the current parsed line has alread been parsed.
# return 1 if timestamp of log line is lower (already parsed)
# return 0 if timestamp from log is upper (line not already parsed)
####
sub incremental_check
{
my ($last_parsed, $old_last_parsed) = @_;
my $current_year = 0;
if ($CONFIG{DEFAULT_YEAR}) {
$current_year = $CONFIG{DEFAULT_YEAR}
} else {
$current_year = (localtime(time))[5]+1900;
}
# set year date part of the current last parsed line from log file
my ($month,$day,@f) = split(/[\s\:]+/, $last_parsed, 5);
$f[0] = sprintf("%02d",$f[0]);
$f[1] = sprintf("%02d",$f[1]);
$f[2] = sprintf("%02d",$f[2]);
my $log_date = $current_year . $MONTH_TO_NUM{"$month"} . sprintf("%02d",$day) . "$f[0]$f[1]$f[2]";
# set year part of the last date from previous run stored in LAST_PARSED file
my ($old_month,$old_day,@old_f) = split(/[\s\:]+/, $old_last_parsed, 5);
$old_f[0] = sprintf("%02d",$old_f[0]);
$old_f[1] = sprintf("%02d",$old_f[1]);
$old_f[2] = sprintf("%02d",$old_f[2]);
my $old_date = $current_year . $MONTH_TO_NUM{"$old_month"} . sprintf("%02d",$old_day) . "$old_f[0]$old_f[1]$old_f[2]";
# Assume that date in the future are in fact logs from previous year so
# substract one year to datetime or use the one given at command line.
if (!$CONFIG{DEFAULT_YEAR}) {
$current_year -= 1;
}
if ($log_date > $CURRENT_TIME) {
$log_date =~ s/^\d{4}//;
$log_date = $current_year . $log_date;
}
if ($old_date > $CURRENT_TIME) {
$old_date =~ s/^\d{4}//;
$old_date = $current_year . $old_date;
}
# The current line has already been parsed. You can not parse data that
# are older than the last run of sendmailanalyzer
return 1 if ($log_date < $old_date);
return 0;
}
# Generate a unique identifier
sub get_uniqueid
{
my $u_id = '';
while (length($u_id) < 16) {
my $c = chr(int(rand(127)));
if ($c =~ /[a-zA-Z0-9]/) {
$u_id .= $c;
}
}
return 'FaKe' . $u_id;
}
sub clean_globals
{
%SYSERR = ();
%DSN = ();
%FROM = ();
%TO = ();
%REJECT = ();
%SPAM = ();
%VIRUS = ();
%ERRMSG = ();
%OTHER = ();
%SPAMDETAIL = ();
%AUTH = ();
%GREYLIST = ();
%POSTGREY = ();
%MSGID = ();
%SKIPMSG = ();
%POSTFIX_PLUGIN_TEMP_DELIVERY = ();
%AMAVIS_ID = ();
%STARTTLS = ();
%SPAMPD = ();
%KEEP_TEMPORARY = ();
%SPF_DKIM = ();
}