#!/usr/bin/perl
#
#    SendmailAnalyzer: maillog parser and statistics reports tool for Sendmail
#    Copyright (C) 2002-2016 Gilles Darold
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
use vars qw($VERSION $AUTHOR $COPYRIGHT $PROGRAM @ARGS);
 
use strict;

use Getopt::Long qw(:config no_ignore_case bundling);
use POSIX qw(:sys_wait_h :errno_h :fcntl_h :signal_h);
use MIME::Base64;
use MIME::QuotedPrint;
use IO::File;


$VERSION     = '9.2';
$AUTHOR = "Gilles Darold <gilles\@darold.net>";
$COPYRIGHT = "(c) 2002-2016 - Gilles Darold <gilles\@darold.net>";

$SIG{'CHLD'} = 'DEFAULT';

# Keep command line arguments in case we received SIGHUP
$PROGRAM = $0;
push(@ARGS, @ARGV);

# Month translation
my %MONTH_TO_NUM = (
	'Jan' => '01',
	'Feb' => '02',
	'Mar' => '03',
	'Apr' => '04',
	'May' => '05',
	'Jun' => '06',
	'Jul' => '07',
	'Aug' => '08',
	'Sep' => '09',
	'Oct' => '10',
	'Nov' => '11',
	'Dec' => '12'
);


# Configuration storage hash
my %CONFIG = ();

# Other configuration directives
my $CONFIG_FILE = "/etc/sendmailanalyzer.conf";
my $SHOW_VER    = 0;
my $INTERACTIVE = 0;
my $HELP        = 0;
my $LAST_PARSE_FILE = 'LAST_PARSED';
my $PID_FILE    = 'sendmailanalyzer.pid';
my $SA_PIPE     = new IO::File;
my $EXIM_REGEX  = qr/^(\d+\-\d+\-\d+) (\d+:\d+:\d+) ([A-Za-z0-9]{6}\-[A-Za-z0-9]{6}\-[A-Za-z0-9]{2}) ([\-\*=><]+) /;
my $HOSTNAME    = '';

# Global variable to store temporary parsed data
my %SYSERR = ();
my %DSN = ();
my %FROM = ();
my %TO = ();
my %REJECT = ();
my %SPAM = ();
my %VIRUS = ();
my $LAST_PARSED = '';
my %ERRMSG = ();
my %OTHER = ();
my %SPAMDETAIL = ();
my %AUTH = ();
my %GREYLIST = ();
my $KID = '';
my %POSTGREY = ();
my %MSGID = ();
my %SKIPMSG = ();
my %POSTFIX_PLUGIN_TEMP_DELIVERY = ();
my %AMAVIS_ID = ();
my %STARTTLS = ();
my %SPAMPD = ();
my %KEEP_TEMPORARY = ();

# Collect command line arguments
GetOptions (
	'a|args=s' => \$CONFIG{TAIL_ARGS},
	'b|break!' => \$CONFIG{BREAK},
	'c|config=s' => \$CONFIG_FILE,
	'd|debug!' => \$CONFIG{DEBUG},
	'f|full!' => \$CONFIG{FULL},
	'F|force!' => \$CONFIG{FORCE},
	'g|postgrey=s' => \$CONFIG{POSTGREY_NAME},
	'h|help!' => \$HELP,
	'i|interactive!' => \$INTERACTIVE,
	'j|journalctl=s' => \$CONFIG{JOURNALCTL_CMD},
	'l|log=s' => \$CONFIG{LOG_FILE},
	'm|mailscanner=s' => \$CONFIG{MAILSCAN_NAME},
	'n|clamd=s' => \$CONFIG{CLAMD_NAME},
	'o|output=s' => \$CONFIG{OUT_DIR},
	'p|piddir=s' => \$CONFIG{PID_DIR},
	's|sendmail=s' => \$CONFIG{MTA_NAME},
	't|tail=s' => \$CONFIG{TAIL_PROG},
	'v|version!' => \$SHOW_VER,
	'w|write-delay=i' => \$CONFIG{DELAY},
	'z|zcat=s' => \$CONFIG{ZCAT_PROG},
	'y|year=s' => \$CONFIG{DEFAULT_YEAR},
	'spamd=s' => \$CONFIG{SPAMD_NAME},
	'hostname=s' => \$HOSTNAME,
);

$CONFIG{FULL} = 1 if ($CONFIG{FORCE});

&usage if ($HELP);

# Read configuration file
&read_config($CONFIG_FILE);

# Checked forced year syntax
if ($CONFIG{DEFAULT_YEAR} && ($CONFIG{DEFAULT_YEAR} !~ /^\d{4}$/)) {
	die "FATAL: Default year $CONFIG{DEFAULT_YEAR} should be 4 digits!\n";
}

# Check if output dir exist and we can write file
if (!-d $CONFIG{OUT_DIR}) {
	die "FATAL: Output directory $CONFIG{OUT_DIR} should exists !\n";
} else {
	open(OUT, ">$CONFIG{OUT_DIR}/test.dat") or die "FATAL: Output directory $CONFIG{OUT_DIR} should be writable !\n";
	close(OUT);
	unlink("$CONFIG{OUT_DIR}/test.dat");
}

if ($CONFIG{DEBUG}) {
       print STDERR "Running in verbose mode...\n";
}       
if ($SHOW_VER || $CONFIG{DEBUG}) {
	print STDERR "\n\tsendmailanalyzer v$VERSION. $COPYRIGHT\n\n";
	exit 0 if ($SHOW_VER);
}

####
# Install signal handlers
####
# Die cleanly on signal
sub terminate
{
	close($SA_PIPE) if ($SA_PIPE >= 0);
	&flush_data(1);
	&dprint("Received terminating signal.", 1);
}

# Restart on signal
sub restart_sa
{
	close($SA_PIPE) if ($SA_PIPE >= 0);
	&dprint("Received SIGHUP signal: reloading configuration file and reopening log file.");
	&flush_data(1);
	&clean_globals();
	exec($^X, $PROGRAM, @ARGS) or die "FATAL: Couldn't restart: $!\n";
}

# Reload configuration and reopen log file on kill -1
my $sigset_hup = POSIX::SigSet->new();
my $action_hup = POSIX::SigAction->new('restart_sa', $sigset_hup, &POSIX::SA_NODEFER);
POSIX::sigaction(&POSIX::SIGHUP, $action_hup);

# Terminate on kill -15
my $sigset_term = POSIX::SigSet->new();
my $action_term = POSIX::SigAction->new('terminate', $sigset_term, &POSIX::SA_NODEFER);
POSIX::sigaction(&POSIX::SIGTERM, $action_term);

# Terminate on kill -9
my $sigset_int = POSIX::SigSet->new();
my $action_int = POSIX::SigAction->new('terminate', $sigset_int, &POSIX::SA_NODEFER);
POSIX::sigaction(&POSIX::SIGKILL, $action_int);

my $CURRENT_TIME = &format_time(localtime(time));

# Run in interactive mode if required
if ($INTERACTIVE) {
	# Start in interactive mode
	print "\n*** sendmailanalyzer v$VERSION (pid:$$) started at " . localtime(time) . "\n";
} else {
	# detach from terminal
	my $pid = fork;
	exit 0 if ($pid);
	die "Couldn't fork: $!" unless defined($pid);
	POSIX::setsid() or die "Can't detach: \$!"; 
	&dprint("Detach from terminal with pid: $$");
}

# Set name of the program without path*
my $orig_name = $0;
$0 = 'sendmailanalyzer';

# Continuously read the maillog file using a pipe to tail program
&dprint("Entering main loop...");
&start_loop;


exit 0;

#-------------------------------- ROUTINES ------------------------------------

####
#  Dump usage to STDERR
####
sub usage
{
	print STDERR qq{
sendmailanalyzer v$VERSION usage:

-a | --args "tail_args": tail command line arguments. Default "-n 0 -f".
-b | --break           : do not run tail after parsing full maillog and exit.
-c | --config file     : path to configuration file. Default is to read it from
                         /etc/sendmailanalyzer.conf.
-d | --debug           : turn on debug mode.
-f | --full            : parse full maillog and compute stat. Default is to read
			 LAST_PARSED file to start from last collected event.
-F | --force           : same as --full but don't take care of LAST_PARSED file,
			 that means that log file always contains new entries.
-h | --help            : show this short help and exit.
-i | --interactive     : run in interactive mode useful if you want day to day
                       : report. Default is daemon mode, real time statistics.
-j | --journalctl cmd  : set the journalctl command to use to replace logfile.
-l | --log file        : path to maillog file. Default is /var/log/maillog.
-m | --mailscanner name: syslog MailScanner program name. Default: Mailscanner.
-o | --output dir      : path to the output directory where data file will be
                         written. Default /var/www/htdocs.
-p | --piddir dir      : path where pid file will be stored. Default /var/run/.
-s | --sendmail name   : syslog sendmail program name. Default sm-mta|sendmail.
-t | --tail tail_prog  : path to the tail system command. Default /usr/bin/tail.
-v | --version         : show version and exit
-w | --write-delay sec : memory storage delay in second before saving data
                         to disk. Default: 5 seconds.
-y | --year 2001       : force the years date part of the log to given value.
			 Default is current year or previous year if log lines
			 appear in the future.
-z | --zcat zcat_prog  : path to the zcat command for compressed maillog.
                         Default /usr/bin/zcat.
     --spamd  name     : syslog Spamd program name. Default: spamd.
     --hostname name   : set a hostname for exim logs. Default: unknown.
};
	exit 0;
}
  
####
# Function used to dump debugging information
####
sub dprint
{
	my $msg = shift;
	my $exit = shift;

	print STDERR "DEBUG: $msg\n" if ($exit || $CONFIG{DEBUG});
	if ($exit) {
		unlink("$CONFIG{PID_DIR}/$PID_FILE");
		exit 1;
	}
}

####
# Start reading maillog file
####
my $OLD_LAST_PARSED = '';
my $OLD_OFFSET      = 0;
sub start_loop
{
	if ($CONFIG{FULL}) {
		if (!$CONFIG{FORCE}) {
			if (-e "$CONFIG{OUT_DIR}/$LAST_PARSE_FILE") {
				if ( not open(IN, "$CONFIG{OUT_DIR}/$LAST_PARSE_FILE")) {
					&logerror("Can't read file $CONFIG{OUT_DIR}/$LAST_PARSE_FILE: $!");
				} else {
					my $tmp = <IN>;
					chomp($tmp);
					($OLD_LAST_PARSED,$OLD_OFFSET) = split(/[\t]/, $tmp);
					close(IN);
				}
			}
		}
		if ($CONFIG{JOURNALCTL_CMD}) {
			$OLD_OFFSET = 0;
			my $since = '';
			if ( ($CONFIG{JOURNALCTL_CMD} !~ /--since|-S/) && ($OLD_LAST_PARSED =~ /^(\d+)-(\d+)-(\d+).(\d+):(\d+):(\d+)/) ) {
				$since = " --since=\"$1-$2-$3 $4:$5:$6\"";
			}
			&dprint("Parsing full entries from command: $CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\"");
			if (!($KID = open(SA_FILE, "$CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\" |"))) {
				&dprint("$0: cannot read input from command: $CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\", $!", 1);
			}
		} else {
			&dprint("Parsing full $CONFIG{LOG_FILE}");
			if ($CONFIG{LOG_FILE} !~ /\.gz/) {
				if (!($KID = open(SA_FILE, "$CONFIG{LOG_FILE}"))) {
					&dprint("$0: cannot read $CONFIG{LOG_FILE}: $!", 1);
				}
			} else {
				# Open a pipe to zcat program for compressed log
				if (!($KID = open(SA_FILE, "$CONFIG{ZCAT_PROG} $CONFIG{LOG_FILE}|"))) {
					&dprint("$0: cannot read from pipe to \"$CONFIG{ZCAT_PROG} $CONFIG{LOG_FILE}\": $!", 1);
				}
			}
		}
		# Write pid file
		if (open(OUT, ">$CONFIG{PID_DIR}/$PID_FILE")) {
			print OUT "$$";
			close(OUT);
		}
		# Move to the last position
		if ($OLD_OFFSET && ($CONFIG{LOG_FILE} !~ /\.gz/)) {
			if ((lstat($CONFIG{LOG_FILE}))[7] < $OLD_OFFSET) {
				&dprint("Log file may have changed, rereading from start of the log file.") if ($CONFIG{DEBUG} > 0);
				$OLD_OFFSET = 0;
			} else {
				&dprint("Jumping to last log offset ($OLD_OFFSET).") if ($CONFIG{DEBUG} > 0);
				my $ret = seek(SA_FILE, $OLD_OFFSET, 0);
				if (!$ret) {
					&dprint("Wrong offset reread from start of the log file.") if ($CONFIG{DEBUG} > 0);
					seek(SA_FILE, 0, 0);
					$OLD_OFFSET = 0;
				}
			}
		} else {
			$OLD_OFFSET = 0;
		}
		while (my $l = <SA_FILE>) {
			chomp($l);
			$l =~ s/ ID \d+ mail.\w//;
			next if ($l =~ /policy-spf|You are still greylisted/);
			$LAST_PARSED = $l;
			$l = '';
			# Only catch relevant logs
			if ( ($LAST_PARSED =~ /($CONFIG{MTA_NAME}|$CONFIG{MAILSCAN_NAME}|$CONFIG{AMAVIS_NAME}|$CONFIG{MD_NAME}|$CONFIG{CLAMD_NAME}|$CONFIG{POSTGREY_NAME}|$CONFIG{SPAMD_NAME}|$CONFIG{CLAMSMTPD_NAME})[\/\[:]/) || ($LAST_PARSED =~ $EXIM_REGEX) ) {
				my $tmpos = tell(SA_FILE);
				if ($OLD_LAST_PARSED) {
					# Store the last position in the log
					# Line already parsed ? If yes, go to retrieve next log line
					if (&incremental_check($LAST_PARSED, $OLD_LAST_PARSED, $OLD_OFFSET) != 0) {
						# Search if this is really the same log file
						if ($OLD_OFFSET && ($tmpos > $OLD_OFFSET)) {
							&dprint("Data are more recent than the one at old offset, maybe the file has changed. Rereading from start of the log file.") if ($CONFIG{DEBUG} > 0);
							seek(SA_FILE, 0, 0);
							$OLD_OFFSET = 0;
							$OLD_LAST_PARSED = ''; 
						}
						next;
					}
					# The line have not already been parsed so erase last date
					# to not go back to this block again
					$OLD_LAST_PARSED = ''; 
				}
				if ($CONFIG{LOG_FILE} !~ /\.gz/) {
					$OLD_OFFSET = $tmpos;
				}
				# Extract common fields, store data in memory and retrieve current time
				my $check_time = &store_data(&parse_common_fields(split(/\s+/, $LAST_PARSED)));
				# Flush data to disk each kind of 5 seconds at least by default
				if ($check_time > $CURRENT_TIME+$CONFIG{DELAY}) {
					$CURRENT_TIME = $check_time;
					&dprint("Flushing data to disk...");
					&flush_data();
				}
			}
		}
		&dprint("Flushing data to disk...");
		&flush_data();
		if ($CONFIG{BREAK}) {
			unlink("$CONFIG{PID_DIR}/$PID_FILE");
			exit 0;
		}
	}

	# Daemon mode is not possible with compressed log file
	if ($CONFIG{LOG_FILE} =~ /\.gz/) {
		&dprint("Daemon mode is not possible with compressed log file", 1);
	}

	# Open a pipe to the tail program or to the journalctl command
	my $since = '';
	if ($CONFIG{JOURNALCTL_CMD} !~ /--since|-S/) {
		if ($LAST_PARSED) {
			if ($LAST_PARSED =~ /^(\d+)-(\d+)-(\d+).(\d+):(\d+):(\d+)/) {
				$since = " --since=\"$1-$2-$3 $4:$5:$6\"";
			}
		} else {
			if ($OLD_LAST_PARSED =~ /^(\d+)-(\d+)-(\d+).(\d+):(\d+):(\d+)/) {
				$since = " --since=\"$1-$2-$3 $4:$5:$6\"";
			}
		}
	}
	if ($CONFIG{JOURNALCTL_CMD}) {
		&dprint("Opening pipe to command: $CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\" -f");
		if (!($KID = open(SA_FILE, "$CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\" -f |"))) {
			&dprint("$0: cannot read input from command: $CONFIG{JOURNALCTL_CMD}$since --output=\"short-iso\" -f, $!", 1);
		}
	} else {
		&dprint("Opening pipe to $CONFIG{TAIL_PROG} $CONFIG{TAIL_ARGS} $CONFIG{LOG_FILE}");
		if ( !($KID = $SA_PIPE->open("$CONFIG{TAIL_PROG} $CONFIG{TAIL_ARGS} $CONFIG{LOG_FILE}|"))) {
			&dprint("$0: cannot read from pipe to \"$CONFIG{TAIL_PROG} $CONFIG{TAIL_ARGS} $CONFIG{LOG_FILE}\": $!");
		}
	}

	# Write pid file
	if (open(OUT, ">$CONFIG{PID_DIR}/$PID_FILE")) {
		print OUT "$$";
		close(OUT);
	}

	# We'll need a non blocking read to be able to intercept signal
	# when there's no new entry in the log file. But for now
	# SIGTERM should be use to interrupt the blocking read.
	while (my $l = <$SA_PIPE>) {
		chomp($l);
		$l =~ s/ ID \d+ mail.\w//;
		next if ($l =~ /policy-spf|You are still greylisted/);
		$LAST_PARSED = $l;
		$l = '';
		# Only catch relevant logs
		my $check_time = '';
		if ( ($LAST_PARSED =~ /($CONFIG{MTA_NAME}|$CONFIG{MAILSCAN_NAME}|$CONFIG{AMAVIS_NAME}|$CONFIG{MD_NAME}|$CONFIG{CLAMD_NAME}|$CONFIG{POSTGREY_NAME}|$CONFIG{SPAMD_NAME})[\/\[]/) || ($LAST_PARSED =~ $EXIM_REGEX) )  {
			# Extract common fields and store data in memory
			$check_time = &store_data(&parse_common_fields(split(/\s+/, $LAST_PARSED)));
		} else {
			$check_time = &format_time(localtime(time));
		}
		# Flush data to disk if write delay is over
		if ($check_time > $CURRENT_TIME+$CONFIG{DELAY}) {
			$CURRENT_TIME = $check_time;
			&dprint("Flushing data to disk ($check_time > $CURRENT_TIME+$CONFIG{DELAY})...");
			&flush_data();
			# Avoid memory leak
			%POSTFIX_PLUGIN_TEMP_DELIVERY = ();
			%AMAVIS_ID = ();
		}
	}

	&dprint("Flushing last data to disk...");
	&flush_data(1);
}

####
# Routine used to extract common field on maillog lines
####
sub parse_common_fields
{
	my ($month,$day,$time,$host,$type,@other) = @_;

	# Get current system time	
	my $ctime = &format_time(localtime(time));

	my $date = '';
	if ($month =~ /(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)/) {
		$date = "$1$2$3";
		unshift(@other, $type);
		unshift(@other, $host);
		$host = $day;
		$type = $time;
		$time = "$4$5$6";
	} elsif ($month =~ /(\d+)-(\d+)-(\d+)/) {
		$date = "$1$2$3";
		unshift(@other, $type);
		$type = $host;
		$host = $time;
		if ($day =~ /(\d+):(\d+):(\d+)/) {
			$time = "$1$2$3";
		}
	} else {
		my $current_year = 0;
		if ($CONFIG{DEFAULT_YEAR}) {
			$current_year = $CONFIG{DEFAULT_YEAR};
		} else {
			$current_year = (localtime(time))[5]+1900;
		}
		$date = $current_year . sprintf("%02d",$MONTH_TO_NUM{"$month"}) . sprintf("%02d",$day);
		my @f = split(/:/, $time);
		$f[0] = sprintf("%02d",$f[0]);
		$f[1] = sprintf("%02d",$f[1]);
		$f[2] = sprintf("%02d",$f[2]);
		$time = "$f[0]$f[1]$f[2]";

		if ("$date$time" > $ctime) {
			# If log timestamp is in the future, use the given one 
			if (!$CONFIG{DEFAULT_YEAR}) {
				$date = ($current_year - 1) . sprintf("%02d",$MONTH_TO_NUM{"$month"}) . sprintf("%02d",$day);
			}
		}
	}
	$type =~ s/\[.*\]\://;

	$host = $CONFIG{MERGING_HOST} if ($CONFIG{MERGING_HOST});

	my $line = join(' ', @other);
	$line =~ s/^\[ID.*\] //;

	return ($date,$time,$host,$type,$line,$ctime);
}

####
# Routine used to store collected data
####
sub store_data
{
	my ($date,$time,$host,$type,$other,$ctime) = @_;

	if ($type =~ /^[<=>\*\-]+$/) {
		&parse_exim("$date","$time",$host,$type,$other);
	} elsif (($CONFIG{MAILSCAN_NAME} || $CONFIG{CLAMD_NAME}) && ($type =~ /^$CONFIG{MAILSCAN_NAME}|$CONFIG{CLAMD_NAME}/i)) {
		&parse_mailscanner("$date","$time",$host,$other);
	} elsif ($CONFIG{AMAVIS_NAME} && ($type =~ /^$CONFIG{AMAVIS_NAME}/i)) {
		&parse_amavis("$date","$time",$host,$other);
	} elsif ($CONFIG{CLAMSMTPD_NAME} && ($type =~ /^$CONFIG{CLAMSMTPD_NAME}/i)) {
		&parse_clamsmtpd("$date","$time",$host,$other);
	} elsif ($CONFIG{MTA_NAME} && ($type =~ /^$CONFIG{MTA_NAME}/i)) {
		&parse_sendmail("$date","$time",$host,$other,$type);
	} elsif ($CONFIG{MD_NAME} && ($type =~ /^$CONFIG{MD_NAME}/i)) {
		&parse_mimedefang("$date","$time",$host,$other);
	} elsif ($CONFIG{POSTGREY_NAME} && ($type =~ /^$CONFIG{POSTGREY_NAME}/i)) {
		&parse_postgrey("$date","$time",$host,$other);
	} elsif ($CONFIG{SPAMD_NAME} && ($type =~ /^$CONFIG{SPAMD_NAME}/i)) {
		&parse_spamd("$date","$time",$host,$other);
	} else {
		&dprint("Skipping unknown syslog report => $date $time $host [$type]: $other") if ($CONFIG{DEBUG} > 1);
	}

	return $ctime;	 
}

sub parse_exim
{

	my ($date,$time,$id,$type,$other) = @_;

	my $time_st = "$date$time";
	my $host = $HOSTNAME || 'unknown';

	if ($type eq '<=') {
		if ($other =~ m#^(.*?) H=(.*?) P=.* S=(\d+) id=(.*)#) {
			my $size = $3;
			my $msgid = $4;
			my $relay = &clean_relay(lc($2));
			$FROM{$host}{$id}{from} = &edecode($1);
			$msgid =~ s/[<>]//g;
			$FROM{$host}{$id}{date} =  $time_st;
			$FROM{$host}{$id}{size} = $size;
			$FROM{$host}{$id}{nrcpts} = 1;
			$FROM{$host}{$id}{msgid} = $msgid;
			$FROM{$host}{$id}{relay} = $relay;
		}
	} elsif ($type =~ /^(=|-)>$/) {
		if ($other =~ m#^(.*?) R=(.*?) T=(.*?) H=(.*)#) {
			my $to = &edecode($1);
			my $relay = &clean_relay(lc($4));
			if ($relay eq $CONFIG{'SKIP_RCPT_RELAY'}) {
				return;
			}
			next if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
			push(@{$TO{$host}{$id}{date}},  $time_st);
			push(@{$TO{$host}{$id}{relay}}, $relay);
			push(@{$TO{$host}{$id}{to}}, $to);
			push(@{$TO{$host}{$id}{status}}, 'Sent');
		}
	} elsif ($type eq '==') {
		if ($other =~ m#^(.*?) R=(.*?) T=([^:]+): (.*)#) {
                        $REJECT{$host}{$id}{relay} = $FROM{$host}{$id}{relay};
                        $REJECT{$host}{$id}{status} = &clear_status($2);
                        $REJECT{$host}{$id}{date} =  $time_st;
                        $REJECT{$host}{$id}{arg1} = &edecode($1);
			my $rule = $3;
			$rule =~ s/\s*\(.*//;
                        $REJECT{$host}{$id}{rule} = $rule;
		}
	} elsif ($type eq '**') {
		if ( !exists $REJECT{$host}{$id}{rule} && ($other =~ m#^([^:]+): (.*)#) ) {
                        $REJECT{$host}{$id}{relay} = $FROM{$host}{$id}{relay};
                        $REJECT{$host}{$id}{rule} = 'error';
                        $REJECT{$host}{$id}{status} = &clear_status($2);
                        $REJECT{$host}{$id}{date} =  $time_st;
                        $REJECT{$host}{$id}{arg1} = &edecode($1);
		}
	}
}

####
# Parse Sendmail syslog output
####
sub parse_sendmail
{
	my ($date,$time,$host,$str,$type) = @_;

	my $time_st = "$date$time";

	# Skip unwanted Postfix lines
	if ($str =~ m#^([^:\s]+): .*#) {
		return if (exists $SKIPMSG{$1});
	}

	#### Store each relevant information per host and id

	# Some spampd line must be skipped
	return if (($type eq 'spampd') && ($str =~ /(processing|clean) message/));

	# Parse MTA system error
	if ($str =~ m#^([^:\s]+): SYSERR[^:]+: (.*)# ) {
		$SYSERR{$host}{$1}{date} =  $time_st;
		$SYSERR{$host}{$1}{message} = &clear_status($2);
	# Skip message related to MCI caching module
	} elsif ($str =~ m#^([^:\s]+): MCI\@#) {
		return;
	# Skip Debug message
	} elsif ($str =~ m#^([^:\s]+):\s+\d+: fl=#) {
		return;
	# POSTFIX: Skip connect/disconnect message
	} elsif ($str =~ m#^(DIS)?CONNECT #i) {
		return;
	# POSTFIX temporary blacklist/whitelist messsage
	} elsif ($str =~ m#^(PASS OLD|PASS NEW|WHITELISTED|BLACKLISTED)#i) {
		return;
	# POSTFIX: Skip postscreen messages
	} elsif ($str =~ m#^(WHITELIST VETO|BARE NEWLINE)#i) {
		return;
	# POSTFIX pregreet test
	} elsif ($str =~ m#^(PREGREET|HANGUP)#i) {
		return;
	# POSTFIX dnsbl message
	} elsif ($str =~ m#^DNSBL rank#i) {
		return;
	# Debug and info messages from POSTFIX
	} elsif  ($str =~ /^(DEBUG|INFO) /) {
		return;
	# POSFIX TLS connexion
	} elsif  ($str =~ /(connect from|setting up TLS connection from)/) {
		return;
	} elsif  ($str =~ /(connect to|setting up TLS connection to|Untrusted TLS connection established)/) {
		return;
	# POSTFIX dnsbl message ???
	} elsif ($str =~ m#addr [a-fA-F0-9\.\:]+ listed#) {
		return;
	# POSTFIX  postscreen messages: COMMAND (PIPELINING|COUNT LIMIT|TIME LIMIT)???
	} elsif ($str =~ m#^COMMAND #i) {
		return;
	# POSTFIX: error messages
	} elsif ($str =~ m#^(lost connection|timeout|too many errors) after ([^\s]+)#) {
		$SYSERR{$host}{"$date$time"}{date} =  $time_st;
		$SYSERR{$host}{"$date$time"}{message} = $1 . ' after ' . $2;
	} elsif ($str =~ m#^connect to [^:]+: (.*)#) {
		$SYSERR{$host}{"$date$time"}{date} =  $time_st;
		$SYSERR{$host}{"$date$time"}{message} = $1;
	} elsif ($str =~ m#^(certificate verification failed for).*:( untrusted issuer| self-signed certificate).*#) {
		$SYSERR{$host}{"$date$time"}{date} =  $time_st;
		$SYSERR{$host}{"$date$time"}{message} = $1 . $2;
	} elsif ($str =~ m#^(SSL_connect error).*#) {
		$SYSERR{$host}{"$date$time"}{date} =  $time_st;
		$SYSERR{$host}{"$date$time"}{message} = $1;
	# Sendmail Milter change subject
	} elsif ($str =~ m#^([^:\s]+): Milter change: header Subject: from (.*?) to (.*)$#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		$FROM{$host}{$id}{subject} = &decode_subject($2);
	# Postfix subject information
	} elsif ($str =~ m#^([^:\s]+): (warning|info): header Subject: (.*?) from ([^;]+);#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		$FROM{$host}{$id}{subject} = &decode_subject($3);
	} elsif ($str =~ m#^warning: .*#) {
		$SYSERR{$host}{"$date$time"}{date} =  $time_st;
		$SYSERR{$host}{"$date$time"}{message} = &clear_status($str);
	# POSTFIX spampd pass
	} elsif ($str =~ m#identified spam (<[^>]+>) \(([^\)]+)\) from ([^\s]+) for ([^\s]+) .* (\d+) bytes.#) {
		my $id = $1;
		my $rule = 'spampd';
		my $from = &edecode($3);
		my $to = &edecode($4);
		my $status = $2;
		my $size = $5;
		foreach my $i (keys %MSGID) {
			if ($i eq $id) {
				$id = $MSGID{$i}{id};
				last;
			}
		}
		return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));

		$SPAM{$host}{$id}{relay} = $FROM{$host}{$id}{relay};
		$SPAM{$host}{$id}{rule} = 'reject';
		$SPAM{$host}{$id}{spam} = $status;
		$SPAM{$host}{$id}{date} =  $time_st;
		$SPAM{$host}{$id}{from} = $from;
		$SPAM{$host}{$id}{to} = $to;
		$SPAM{$host}{$id}{status} = $status;

	# POSTFIX spampd pass
	} elsif ($str =~ m#identified spam \(([^\)]+)\) \(([^\)]+)\) from ([^\s]+) for ([^\s]+) .* (\d+) bytes.#) {
		my $id = $1;
		my $rule = 'spampd';
		my $from = &edecode($3);
		my $to = &edecode($4);
		my $status = $2;
		my $size = $5;
		return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));

		$status =~ s/.*\///;
		$SPAMPD{$host}{$from}{$to}{rule} = 'reject';
		$SPAMPD{$host}{$from}{$to}{spam} = $status;
		$SPAMPD{$host}{$from}{$to}{date} =  $time_st;
		$SPAMPD{$host}{$from}{$to}{status} = $status;

	# Sendmail authid single message
	} elsif ($str =~ m#^([^:\s]+): authid=#) {
		return;
	# POSTFIX remove message id
	} elsif ($str =~ m#^([^:\s]+): removed$#) {
		my $id = $1;
		foreach (keys %MSGID) {
			delete $MSGID{$_} if ($MSGID{$_}{id} eq $id);
		}
		delete $SKIPMSG{$id};
		delete $KEEP_TEMPORARY{$id};
		return;
	# Sendmail subject information
	} elsif ($str =~ m#^([^:\s]+): Subject:(.*)#) {
		my $id = $1;
		$FROM{$host}{$id}{subject} =  &decode_subject($2);
	# Parse protocole error
	} elsif ($str =~ m#^([^:\s]+): ([^:\s]+): (.*protocol error:.*)#) {
		$SYSERR{$host}{$1}{date} =  $time_st;
		$SYSERR{$host}{$1}{message} = $3;
	# Parse virus found by clamav-milter
	} elsif ($str =~ m#^([^:\s]+): Milter add: header: X-Virus-Status: Infected with (.*)#) {
		$VIRUS{$host}{$1}{virus} = $2;
		$VIRUS{$host}{$1}{file} = 'Inline';
		$VIRUS{$host}{$1}{date} =  $time_st;
	} elsif ($str =~ m#^([^:\s]+): Milter [^:]+: header: X-Virus-Status: Infected \((.*)\)#) {
		$VIRUS{$host}{$1}{virus} = $2;
		$VIRUS{$host}{$1}{file} = 'Inline';
		$VIRUS{$host}{$1}{date} =  $time_st;
	# Parse spam found by spamd-milter
	} elsif ($str =~ m#^([^:\s]+): Milter (add|change): header: X-Spam-Status: Yes, score=([^\s]+) required=([^\s]+) tests=([^\s]+)#) {
		my $id = $1;
		$SPAM{$host}{$id}{spam} = 'spamdmilter';
		$SPAM{$host}{$id}{date} =  $time_st;
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$id}{date} =  $time_st;
			$SPAMDETAIL{$host}{$id}{type} = 'spamdmilter';
			$SPAMDETAIL{$host}{$id}{spam} = $5;
			$SPAMDETAIL{$host}{$id}{required} = $4;
			$SPAMDETAIL{$host}{$id}{score} = $3;
			if ($SPAMDETAIL{$host}{$id}{spam} =~ s/(nt|\s)autolearn=([^\s,]+)//) {
				$SPAMDETAIL{$host}{$id}{autolearn} = $2;
			}
			$SPAMDETAIL{$host}{$id}{spam} =~ s/,nt//g;
		}
	# Parse spam found by postfix/policyd-weight
	} elsif ($str =~ m#decided action=DUNNO\s+(.*); rate: ([^;]+); <client=([^>]+)> <helo=([^>]+)> <from=([^>]+)> <to=([^>]+)>; delay.*#) {
		my $spam = $1;
		my $score = $2;
		my $relay = $3;
		my $from = $5;
		my $to = $6;
		my $id = &get_uniqueid();

		$SPAM{$host}{$id}{spam} = 'policydweight';
		$SPAM{$host}{$id}{date} =  $time_st;
		$SPAM{$host}{$id}{relay} = &clean_relay($relay);
		$SPAM{$host}{$id}{from} = &edecode($from);
		$SPAM{$host}{$id}{to} = &edecode($to);
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$id}{date} =  $time_st;
			$SPAMDETAIL{$host}{$id}{type} = 'policydweight';
			$SPAMDETAIL{$host}{$id}{spam} = $spam;
			$SPAMDETAIL{$host}{$id}{score} = $score;
		}
	# Parse spam found in cache by postfix/policyd-weight
	} elsif ($str =~ m#decided action=DUNNO\s+(.*); rate: ([^;]+); <client=([^>]+)> <helo=([^>]+)> <from=([^>]+)> <to=([^>]+)>; delay.*#) {
		my $spam = $1;
		my $score = $2;
		my $relay = $3;
		my $from = $5;
		my $to = $6;
		my $id = &get_uniqueid();
		$SPAM{$host}{$id}{spam} = 'policydweight';
		$SPAM{$host}{$id}{date} =  $time_st;
		$SPAM{$host}{$id}{relay} = &clean_relay($relay);
		$SPAM{$host}{$id}{from} = &edecode($from);
		$SPAM{$host}{$id}{to} = &edecode($to);
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$id}{date} =  $time_st;
			$SPAMDETAIL{$host}{$id}{type} = 'policydweight';
			$SPAMDETAIL{$host}{$id}{spam} = $spam;
			$SPAMDETAIL{$host}{$id}{score} = $score;
		}
	# Parse spam found by dnsbl-milter
	} elsif ($str =~ m#^([^:\s]+): Milter: from=([^,]+), reject=(.*)#) {
		my $id = $1;
		my $spam = $3;
		$spam =~ s/ See .*//;
		$spam =~ s/\/.*//;
		$SPAM{$host}{$id}{spam} = 'dnsblmilter';
		$SPAM{$host}{$id}{date} =  $time_st;
		$SPAM{$host}{$id}{from} = &edecode($2);
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$id}{date} =  $time_st;
			$SPAMDETAIL{$host}{$id}{type} = 'dnsblmilter';
			$SPAMDETAIL{$host}{$id}{spam} = $spam;
		}
	# Parse spam found by jchkmail
	} elsif ($str =~ m#^([^:\s]+): Milter (add|change): header: X-j-chkmail-Status: (Spam|Unsure)(.*)#) {
		$SPAM{$host}{$1}{spam} = 'jchkmail';
		$SPAM{$host}{$1}{date} =  $time_st;
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$1}{type} = 'jchkmail';
			$SPAMDETAIL{$host}{$1}{spam} = $3 . $4;
			$SPAMDETAIL{$host}{$1}{date} =  $time_st;
		}
	} elsif ($str =~ m#^([^:\s]+): Milter (add|change): header: X-j-chkmail-Score: .*> S=(.*)#) {
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$1}{type} = 'jchkmail';
			$SPAMDETAIL{$host}{$1}{score} = $3;
		}
	} elsif ($str =~ m#^([^:\s]+): Milter (delete|add|change): #) {
		# Skip other milter header modication notice
		return;
	} elsif ($str =~ m#^([^:\s]+): Milter insert \(\d+\): #) {
		# Skip other milter header insertion notice
		return;
	} elsif ($str =~ m#^([^:\s]+): Milter: connect: .*, ([^,]*)#) {
		$SYSERR{$host}{$1}{date} =  $time_st;
		$SYSERR{$host}{$1}{message} = &clear_status($2);
	} elsif ($str =~ m#^([^:\s]+): Milter (delete|add|change) #) {
		# Skip other milter modication notice
		return;
	} elsif ($str =~ m#^([^:\s]+): Milter: data, reject=(.*)#) {
		my $id = $1;
		my $status = $2;
		if ($status =~ /Blocked by SpamAssassin/) {
			$SPAM{$host}{$id}{spam} = 'Blocked by SpamAssassin';
		} elsif ($status =~ /554 5\.7\.1 (.*) detected by ClamAV.*/) {
			$SPAM{$host}{$id}{spam} = "ClamAv $1"
		} elsif ($status =~ /550 5\.7\.0 (.*) id=/) {
			$SPAM{$host}{$id}{spam} = "eXpurGate $1"
		} elsif ($status =~ /554 5\.7\.1 Command rejected/) {
			return # Already handle at Clamav virus report
		} elsif ($status =~ /Please try again later/) {
			return; # Skip greylist rejection
		} else {
			$SPAM{$host}{$id}{spam} = $status;
		}
	# Local failure
	} elsif ($str =~ m#^([^:\s]+): <([^>]+)>\.\.\. (.*)#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		my $to = $2;
		my $status = &clear_status($3);
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		$to = &edecode($to);
		return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
		push(@{$TO{$host}{$id}{to}}, $to);
		push(@{$TO{$host}{$id}{date}},  $time_st);
		push(@{$TO{$host}{$id}{status}}, $status);
	# POSTFIX queue From clause
	} elsif ($str =~ m#^([^:\s]+): from=([^,]*), size=(\d+), nrcpt=(\d+).*#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		my $from = lc($2);
		$from ||= 'empty';
		my $size = $3;
		my $nrcpts = $4;
		$FROM{$host}{$id}{date} =  $time_st;
		$FROM{$host}{$id}{from} = &edecode($from);
		$FROM{$host}{$id}{size} = $size;
		$FROM{$host}{$id}{nrcpts} = $nrcpts;
		if (exists $SPAMPD{$host}{$from}) {
			foreach my $to (keys %{$SPAMPD{$host}{$from}}) {

				$SPAM{$host}{$id}{relay} = $FROM{$host}{$id}{relay};
				$SPAM{$host}{$id}{rule} = $SPAMPD{$host}{$from}{$to}{rule};
				$SPAM{$host}{$id}{spam} = $SPAMPD{$host}{$from}{$to}{spam};
				$SPAM{$host}{$id}{date} = $SPAMPD{$host}{$from}{$to}{date};
				$SPAM{$host}{$id}{status} = $SPAMPD{$host}{$from}{$to}{status};
			}
			delete $SPAMPD{$host}{$from};
		}
	# Catch POSTFIX SASL AUTH method
	} elsif ($str =~ m#^([^:\s]+): client=([^,]+), sasl_method=([^,]+), sasl_username=(.*)#) {
		my $authid = $4;
		push(@{$AUTH{$host}{$authid}{type}}, 'SASL');
		push(@{$AUTH{$host}{$authid}{mech}}, $3);
		push(@{$AUTH{$host}{$authid}{date}},  $time_st);
		push(@{$AUTH{$host}{$authid}{relay}}, &clean_relay(lc($2)));
	# POSTFIX client origin and relay
	} elsif ($str =~ m#^([^:\s]+): client=([^,]*)#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		my $relay = &clean_relay(lc($2));

		# POSTFIX origin queue
		# Get real origin when available
		if ($str =~ m#^[^:\s]+: client=([^,]+), orig_queue_id=([^,]+), orig_client=([^,]+)#) {
			$KEEP_TEMPORARY{$id} = $2;
			$id = $2;
			$relay = &clean_relay(lc($3));
		}
		$FROM{$host}{$id}{relay} = $relay;
		# Catch POSTFIX SASL SMTP AUTH
		if ($str =~ m#sasl_method=([^,]*), sasl_username=([^,]*)#) {
			my $authid = $2;
			push(@{$AUTH{$host}{$authid}{type}}, 'SASL');
			push(@{$AUTH{$host}{$authid}{mech}}, $1);
			push(@{$AUTH{$host}{$authid}{date}},  $time_st);
			push(@{$AUTH{$host}{$authid}{relay}}, $relay);
		}
	# POSTFIX message id
	} elsif ($str =~ m#^([^:\s]+): message-id=([^,]*)#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		my $msgid = $2;
		$msgid =~ s/[<>]//g;
		if (exists $MSGID{$msgid}) {
			$KEEP_TEMPORARY{$id} = $MSGID{$msgid}{id};
		}
		$MSGID{$msgid}{id} = $id;
		$FROM{$host}{$id}{msgid} = $msgid;
		return;
	# POSTFIX local relay
	} elsif ($str =~ m#^([^:\s]+): uid=\d+ from=#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		$FROM{$host}{$id}{relay} = 'localhost';
	# From clause
	} elsif ($str =~ m#^([^:\s]+): from=([^,]*), size=(\d+),.*, nrcpts=(\d+), msgid=([^,]+),.*relay=([^,]+)#) {
		my $id = $1;
		my $from = lc($2);
		$from ||= 'empty';
		my $size = $3;
		my $nrcpts = $4;
		my $msgid = $5;
		my $relay = &clean_relay(lc($6));
		$msgid =~ s/[<>]//g;
		$FROM{$host}{$id}{date} =  $time_st;
		$FROM{$host}{$id}{from} = &edecode($from);
		$FROM{$host}{$id}{size} = $size;
		$FROM{$host}{$id}{nrcpts} = $nrcpts;
		$FROM{$host}{$id}{msgid} = $msgid;
		$FROM{$host}{$id}{relay} = $relay;
	} elsif ($str =~ m#^([^:\s]+): from=([^,]*), size=(\d+),.*, nrcpts=(\d+),.*relay=([^,]+)#) {
		my $id = $1;
		my $from = lc($2);
		$from ||= 'empty';
		my $size = $3;
		my $nrcpts = $4;
		my $relay = &clean_relay(lc($5));
		if (exists $GREYLIST{$id}) {
			delete $GREYLIST{$id};
			return;
		} elsif (exists $REJECT{$host}{$id} && !$size) {
			$REJECT{$host}{$id}{arg1} = &edecode($from);
			$REJECT{$host}{$id}{relay} = $relay;
		}
		$FROM{$host}{$id}{date} =  $time_st;
		$FROM{$host}{$id}{from} = &edecode($from);
		$FROM{$host}{$id}{size} = $size;
		$FROM{$host}{$id}{nrcpts} = $nrcpts;
		$FROM{$host}{$id}{relay} = $relay;

	# POSTFIX To clause 
	} elsif ($str =~ m#^([^:\s]+): to=([^,]+), relay=([^,]+),.*status=(.*)#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		my $to = &edecode($2);
		my $relay = &clean_relay(lc($3));
		my $status = $4;
		if ($status =~ /queued as ([^\)]+)\)/) {
			$KEEP_TEMPORARY{$1} = $id;
			return;
		}
		# Spam message discarded by amavisd and reported as sent must be affected to spam
		if ($status =~ /sent \(250 [^,]+, ([^,]+), .* spam\)/) {
			$status = ucfirst($1);
			$SPAM{$host}{$id}{relay} = $relay;
			$SPAM{$host}{$id}{rule} = 'discarded';
			$SPAM{$host}{$id}{spam} = "Amavis $status spam";
			$SPAM{$host}{$id}{date} =  $time_st;
			$SPAM{$host}{$id}{from} = $FROM{$host}{$id}{from} if (exists $FROM{$host}{$id}{from});
			if ($CONFIG{SPAM_DETAIL}) {
				$SPAMDETAIL{$host}{$id}{date} =  $time_st;
				$SPAMDETAIL{$host}{$id}{type} = 'amavis';
				$SPAMDETAIL{$host}{$id}{spam} = $status;
			}
			return;
		}
		# Virus message discarded by amavisd and reported as sent must be affected to virus
		if ($status =~ /sent \(250 [^,]+, ([^,]+), .* INFECTED: (.*)\)/) {
			$status = ucfirst($1);
			$VIRUS{$host}{$id}{file} = 'Inline';
			$VIRUS{$host}{$id}{virus} = $2;
			$VIRUS{$host}{$id}{relay} = $relay;
			$VIRUS{$host}{$id}{from} = $FROM{$host}{$id}{from} if (exists $FROM{$host}{$id}{from});
			$VIRUS{$host}{$id}{to} = $to;
			$VIRUS{$host}{$id}{date} =  $time_st;
			return;
		}
		if ($status =~ /sent \(250 OK, sent [^\s]+ ([^\)]+)\)/) {
			$KEEP_TEMPORARY{$1} = $id;
			return;
		}
		$status = &clear_status($status, $id);
		if ($relay eq $CONFIG{'SKIP_RCPT_RELAY'}) {
			return;
		}
		# Store DSN id mapping
		if ($status eq 'Bounced') {
			$DSN{$host}{$id}{srcid} = $id;
			$DSN{$host}{$id}{status} = $status;
			$DSN{$host}{$id}{date} =  $time_st;
		} else {
			$status = 'Sent' if ($status eq 'sent');
		}
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		foreach my $t (split(/,/, $to)) {
			$t = &edecode($t);
			return if ($t eq 'more');
			next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
			push(@{$TO{$host}{$id}{date}},  $time_st);
			push(@{$TO{$host}{$id}{relay}}, $relay);
			push(@{$TO{$host}{$id}{to}}, $t);
			push(@{$TO{$host}{$id}{status}}, $status);
		}

	# POSTFIX To clause with origine
	} elsif ($str =~ m#^([^:\s]+): to=([^,]+), orig_to=([^,]+), relay=([^,]+),.*status=(.*)#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		my $to = &edecode($2);
		my $relay = &clean_relay(lc($4));
		if ($relay eq $CONFIG{'SKIP_RCPT_RELAY'}) {
			return;
		}
		my $status = &clear_status($5, $id);
		$status = 'Sent' if ($status eq 'sent'); 
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		foreach my $t (split(/,/, $to)) {
			$t = &edecode($t);
			return if ($t eq 'more');
			next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
			push(@{$TO{$host}{$id}{to}}, $t);
			push(@{$TO{$host}{$id}{status}}, $status);
		}
	# To clause queued
	} elsif ($str =~ m#^([^:\s]+): to=(.*), delay=.* stat=queued#) {
		my $id = $1;
		my $to = &edecode($2);
		return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
		push(@{$TO{$host}{$id}{queue_date}},  $time_st);
		push(@{$TO{$host}{$id}{queue_to}}, $to);
	# To clause generated by a mailing list
	} elsif ($str =~ m#^([^:\s]+): to=(.*), ctladdr=([^\s]+).*, mailer=prog,.*stat=(.*)#) {
		my $id = $1;
		my $prog = lc($2);
		my $ctladdr = &edecode($3);
		my $status = &clear_status($4);
		$prog =~ s/\|//g;
		# Skip vacation and procmail prog that double the recipient entry
		return if ($prog =~ /(vacation|procmail)/);
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		return if ($CONFIG{EXCLUDE_TO} && ($ctladdr =~ /^$CONFIG{EXCLUDE_TO}$/));
		push(@{$TO{$host}{$id}{date}},  $time_st);
		push(@{$TO{$host}{$id}{relay}}, 'localhost');
		push(@{$TO{$host}{$id}{to}}, $ctladdr);
		push(@{$TO{$host}{$id}{status}}, $status);
	} elsif ($str =~ m#^([^:\s]+): to=(.*), ctladdr=([^\s]+).*, mailer=\*file\*,.*, stat=(.*)#) {
		my $id = $1;
		my $prog = lc($2);
		my $ctladdr = &edecode($3);
		my $status = &clear_status($4);
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		return if ($CONFIG{EXCLUDE_TO} && ($ctladdr =~ /^$CONFIG{EXCLUDE_TO}$/));
		push(@{$TO{$host}{$id}{date}},  $time_st);
		push(@{$TO{$host}{$id}{relay}}, 'localhost');
		push(@{$TO{$host}{$id}{to}}, $ctladdr);
		push(@{$TO{$host}{$id}{status}}, $status);
	} elsif ($str =~ m#^([^:\s]+): to=(.*), ctladdr=([^\s]+).*, mailer=local, .*, stat=(.*)#) {
		my $id = $1;
		my $to = &edecode($2);
		my $ctladdr = &edecode($3);
		my $status = &clear_status($4);
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		foreach my $t (split(/,/, $to)) {
			$t = &edecode($t);
			return if ($t eq 'more');
			next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
			push(@{$TO{$host}{$id}{date}},  $time_st);
			push(@{$TO{$host}{$id}{relay}}, 'localhost');
			push(@{$TO{$host}{$id}{to}}, $t);
			push(@{$TO{$host}{$id}{status}}, $status);
		}

	# To clause generated by a redirection
	} elsif ($str =~ m#^([^:\s]+): to=(.*), ctladdr=([^\s]+).*, mailer=.*, relay=([^,]+),.*stat=(.*)#) {
		my $id = $1;
		my $to = &edecode($2);
		my $ctladdr = &edecode($3);
		my $relay = &clean_relay(lc($4));
		if ($relay eq $CONFIG{'SKIP_RCPT_RELAY'}) {
			return;
		}
		my $status = &clear_status($5);
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		foreach my $t (split(/,/, $to)) {
			$t = &edecode($t);
			return if ($t eq 'more');
			next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
			push(@{$TO{$host}{$id}{date}},  $time_st);
			push(@{$TO{$host}{$id}{relay}}, 'localhost');
			push(@{$TO{$host}{$id}{to}}, $t);
			push(@{$TO{$host}{$id}{status}}, $status);
		}
	# To close of intercepted virus by clamav-milter
	} elsif ($str =~ m#^([^:\s]+): to=(.*), delay=.*, stat=(virus .*) detected by#) {
		my $id = $1;
		my $to = $2;
		my $status = &clear_status($3);
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		foreach my $t (split(/,/, $to)) {
			$t = &edecode($t);
			return if ($t eq 'more');
			next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
			push(@{$TO{$host}{$id}{date}},  $time_st);
			push(@{$TO{$host}{$id}{to}}, $t);
			push(@{$TO{$host}{$id}{status}}, $status);
		}
	# To clause with local distribution
	} elsif ($str =~ m#^([^:\s]+): to=(.*), delay=.*, mailer=local,.*, stat=(.*)#) {
		my $id = $1;
		my $to = $2;
		my $status = &clear_status($3);
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		foreach my $t (split(/,/, $to)) {
			$t = &edecode($t);
			return if ($t eq 'more');
			next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
			push(@{$TO{$host}{$id}{date}},  $time_st);
			push(@{$TO{$host}{$id}{relay}}, 'localhost');
			push(@{$TO{$host}{$id}{to}}, $t);
			push(@{$TO{$host}{$id}{status}}, $status);
		}
	# To clause
	} elsif ($str =~ m#^([^:\s]+): to=(.*), delay=.*, relay=([^,]+),.*stat=(.*)#) {
		my $id = $1;
		my $to = $2;
		my $relay = &clean_relay(lc($3));
		my $status = &clear_status($4);
		if ($relay eq $CONFIG{'SKIP_RCPT_RELAY'}) {
			return;
		}
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		foreach my $t (split(/,/, $to)) {
			$t = &edecode($t);
			return if ($t eq 'more');
			next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
			push(@{$TO{$host}{$id}{date}},  $time_st);
			push(@{$TO{$host}{$id}{relay}}, $relay);
			push(@{$TO{$host}{$id}{to}}, $t);
			push(@{$TO{$host}{$id}{status}}, $status);
		}
	# To clause with no delivery. Most of the time follow a reject.
	} elsif ($str =~ m#^([^:\s]+): to=(.*), delay=.*, pri=([^,]+), stat=(.*)#) {
		my $id = $1;
		my $to = $2;
		my $status = &clear_status($4);
		return if (exists $REJECT{$host}{$id});
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		foreach my $t (split(/,/, $to)) {
			$t = &edecode($t);
			return if ($t eq 'more');
			next if ($CONFIG{EXCLUDE_TO} && ($t =~ /^$CONFIG{EXCLUDE_TO}$/));
			push(@{$TO{$host}{$id}{date}},  $time_st);
			push(@{$TO{$host}{$id}{to}}, $t);
			push(@{$TO{$host}{$id}{status}}, $status);
		}

	# Ruleset reject clause
	} elsif ($str =~ m#^([^:\s]+): ruleset=([^,]+), arg1=([^,]+), relay=([^,]+),.*reject=(.*)#) {
		my $id = $1;
		my $rule = $2;
		my $arg1 = $3;
		my $relay = &clean_relay(lc($4));
		my $reject = $5;
		$arg1 =~ s/[<>]+//g;
		# Test Sendmail DNSBL spam scan
		if (($reject =~ /553 5\.3\.0/i) || ($reject =~ /550 5\.7\.1/i)) {
			$SPAM{$host}{$id}{relay} = $relay;
			$SPAM{$host}{$id}{rule} = $rule;
			$SPAM{$host}{$id}{spam} = 'DNSBL Spam blocked';
			$SPAM{$host}{$id}{date} =  $time_st;
			$SPAM{$host}{$id}{from} = $arg1;
			if ($CONFIG{SPAM_DETAIL}) {
				$SPAMDETAIL{$host}{$id}{date} =  $time_st;
				$SPAMDETAIL{$host}{$id}{type} = 'dnsbl';
				$reject =~ s/.*(Spam blocked see: )/$1/;
				$reject =~ s/.*(Spam blocked .* found in .*)/$1/;
				$reject =~ s/.*Rejected: .* (listed at [^\s]+).*/$1/;
				$SPAMDETAIL{$host}{$id}{spam} = &clear_status($reject);
			}
		} else {
			$REJECT{$host}{$id}{relay} = $relay;
			$REJECT{$host}{$id}{rule} = $rule;
			$REJECT{$host}{$id}{status} = &clear_status($reject);
			$REJECT{$host}{$id}{date} =  $time_st;
			$REJECT{$host}{$id}{arg1} = &edecode($arg1);
		}

	# Ruleset reject clause
	} elsif ($str =~ m#^ruleset=([^,]+), arg1=([^,]+),.* relay=([^,]+),.*reject=(.*)#) {
		my $rule = $1;
		my $arg1 = &clean_relay(lc($2));
		my $relay = &clean_relay(lc($3));
		my $reject = $4;
		$arg1 =~ s/[<>]+//g;
		my $id = &get_uniqueid();
		# Test Sendmail DNSBL spam scan
		if ($reject =~ /(553 5\.3\.0|550 5\.7\.1)/i) {
			$SPAM{$host}{$id}{relay} = $relay;
			$SPAM{$host}{$id}{rule} = $rule;
			$SPAM{$host}{$id}{spam} = 'DNSBL Spam blocked';
			$SPAM{$host}{$id}{date} =  $time_st;
			$SPAM{$host}{$id}{from} = $arg1;
			if ($CONFIG{SPAM_DETAIL}) {
				$SPAMDETAIL{$host}{$id}{date} =  $time_st;
				$SPAMDETAIL{$host}{$id}{type} = 'dnsbl';
				$reject =~ s/.*(Spam blocked see: )/$1/;
				$reject =~ s/.*(Spam blocked .* found in .*)/$1/;
				$SPAMDETAIL{$host}{$id}{spam} = $reject;
			}
		} else {
			$REJECT{$host}{$id}{relay} = $relay;
			$REJECT{$host}{$id}{rule} = $rule;
			$REJECT{$host}{$id}{status} = &clear_status($reject);
			$REJECT{$host}{$id}{date} =  $time_st;
			$REJECT{$host}{$id}{arg1} = &edecode($arg1);
		}

	# Add support to milter recipient rejection report
	} elsif ($str =~ m#^([^:\s]+): Milter: to=(.*), reject=(\d+ \d+\.\d+\.\d+\s+.*)#) {
		my $id = $1;
		my $status = $3;
		my $to = &edecode($2);
		return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
		if ($status =~ /Greylisting in action/) {
			$GREYLIST{$id} = $to;
			return;
		}
		$status = &clear_status($status);
		push(@{$TO{$host}{$id}{date}},  $time_st);
		push(@{$TO{$host}{$id}{to}}, $to);
		push(@{$TO{$host}{$id}{status}}, $status);
		$REJECT{$host}{$id}{rule} = 'reject';
		$REJECT{$host}{$id}{status} = $status;
		$REJECT{$host}{$id}{date} =  $time_st;
	# Add support to milter sender rejection
	} elsif ($str =~ m#^([^:\s]+): Milter: from=(.*), reject=(\d+ \d+\.\d+\.\d+\s+.*)#) {
		my $id = $1;
		my $arg1 = $2;
		my $reject = $3;
		$arg1 =~ s/[<>]+//g;
		if ($reject =~ /sender=(.*)\&ip=(.*)\&receiver=/) {
			$REJECT{$host}{$id}{arg1} = &edecode($1);
			$REJECT{$host}{$id}{relay} = $2;
		} else {
			$REJECT{$host}{$id}{arg1} = $arg1;
		}
		$REJECT{$host}{$id}{rule} = 'reject' if (!$REJECT{$host}{$id}{rule});
		$REJECT{$host}{$id}{status} = &clear_status($reject);
		$REJECT{$host}{$id}{date} =  $time_st;

	# Parse virus quarantined by clamav-milter
	} elsif ($str =~ m#^([^:\s]+): milter=clamav-milter, quarantine=quarantined by clamav-milter#) {
		if (!exists $VIRUS{$host}{$1}{virus}) {
			$VIRUS{$host}{$1}{virus} = 'Quarantined by clamav-milter';
			$VIRUS{$host}{$1}{file} = 'Inline';
			$VIRUS{$host}{$1}{date} =  $time_st;
		} else {
			$VIRUS{$host}{$1}{virus} .= ' - Quarantined by clamav-milter';
		}

	# Try to find milter rejection rule (other fields will be overriden by the condition above)
	} elsif ($str =~ m#^([^:\s]+): milter=([^,]+), action=([^,]+), reject=(\d+ \d+\.\d+\.\d+\s+.*)#) {
		my $id = $1;
		my $reject = $4;
		$REJECT{$host}{$id}{rule} = $2;
		$REJECT{$host}{$id}{action} = $3;
		$REJECT{$host}{$id}{status} = &clear_status($reject);
		$REJECT{$host}{$id}{date} =  $time_st;
		if ($reject =~ /sender=(.*)\&ip=(.*)\&receiver=/) {
			$REJECT{$host}{$id}{arg1} = &edecode($1);
			$REJECT{$host}{$id}{relay} = $2;
		}

	# Store DSN id mapping
	} elsif ($str =~ m#^([^:\s]+): ([^:\s]+): (DSN|return to sender|sender notify|postmaster notify): (.*)# ) {
		my $previd = $1;
		my $id = $2;
		my $type = $3;
		my $status = $4;
		$status =~ s/(.*)\.\.\. //;
		$DSN{$host}{$id}{srcid} = $previd;
		$DSN{$host}{$id}{status} = &clear_status($type . " " . $status);
		$DSN{$host}{$id}{date} =  $time_st;
		$DSN{$host}{$id}{status} =~ s/ \((.*?)\)//g;
		$FROM{$host}{$id}{date} =  $time_st;
		$FROM{$host}{$id}{from} = 'DSN@localhost';
		$FROM{$host}{$id}{size} = 0;
		$FROM{$host}{$id}{nrcpts} = 1;
		$FROM{$host}{$id}{relay} = 'localhost';
	# POSTFIX reject messages
	} elsif ($str =~ m#^([^:\s]+): reject: RCPT from ([^\s]+) (.*) from=<([^>]*)>[,]* to=<([^>]+)>#) {
		my $reject = $3;
		my $from = $4;
		my $to = $5;
		my $relay = &clean_relay(lc($2));
		my $id = &get_uniqueid();
		$reject =~ s/^([^;]+);//;
		my $status = $1 || '';
		$reject =~ s/^\s+//;
		$reject =~ s/[\s;]+$//;
		# Test PostFix DNSBL spam scan
		if ($reject =~ /client .* (blocked using .*)/i) {
			my $spamdetail = $1;
			$SPAM{$host}{$id}{relay} = $relay;
			$SPAM{$host}{$id}{rule} = 'reject';
			$SPAM{$host}{$id}{spam} = 'DNSBL Spam blocked';
			$SPAM{$host}{$id}{date} =  $time_st;
			$SPAM{$host}{$id}{from} = &edecode($from);
			$SPAM{$host}{$id}{to} = &edecode($to);
			$SPAM{$host}{$id}{status} = &clear_status($status);
			if ($CONFIG{SPAM_DETAIL}) {
				$SPAMDETAIL{$host}{$id}{date} =  $time_st;
				$SPAMDETAIL{$host}{$id}{type} = 'dnsbl';
				$SPAMDETAIL{$host}{$id}{spam} = &clear_status($spamdetail);
			}
		} elsif ($status =~ /(.* spam.*)/i) {
			$SPAM{$host}{$id}{relay} = $relay;
			$SPAM{$host}{$id}{rule} = 'reject';
			$SPAM{$host}{$id}{spam} = 'Spam blocked';
			$SPAM{$host}{$id}{date} =  $time_st;
			$SPAM{$host}{$id}{from} = &edecode($from);
			$SPAM{$host}{$id}{to} = &edecode($to);
			$SPAM{$host}{$id}{status} = &clear_status($status);
			if ($CONFIG{SPAM_DETAIL}) {
				$SPAMDETAIL{$host}{$id}{date} =  $time_st;
				$SPAMDETAIL{$host}{$id}{type} = 'dnsbl';
				$SPAMDETAIL{$host}{$id}{spam} = &clear_status($status);
			}
		} else {
			$REJECT{$host}{$id}{relay} = $relay;
			$REJECT{$host}{$id}{rule} = 'reject';
			$REJECT{$host}{$id}{status} = &clear_status($status);
			$REJECT{$host}{$id}{date} =  $time_st;
			$REJECT{$host}{$id}{arg1} = &edecode($from);
			$REJECT{$host}{$id}{to} = $to;
		}

	# POSTFIX spampd reject
	} elsif ($str =~ m#^([^:\s]+): reject: header X-Spam-Flag: YES from ([^;]+); from=<([^>]*)> to=<([^>]+)> [^:]+: (.*)#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		my $relay = &clean_relay(lc($2));
		my $rule = 'spampd';
		my $from = &edecode($3);
		my $to = &edecode($4);
		my $status = &clear_status($5);
		return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
		$SPAM{$host}{$id}{relay} = $relay;
		$SPAM{$host}{$id}{rule} = 'reject';
		$SPAM{$host}{$id}{spam} = $status;
		$SPAM{$host}{$id}{date} =  $time_st;
		$SPAM{$host}{$id}{from} = $from;
		$SPAM{$host}{$id}{to} = $to;
		$SPAM{$host}{$id}{status} = $status;
		if (!exists $FROM{$host}{$id}{date}) {
			$FROM{$host}{$id}{date} =  $time_st;
			$FROM{$host}{$id}{from} = $from;
			$FROM{$host}{$id}{size} = 0;
			$FROM{$host}{$id}{nrcpts} = 1;
			$FROM{$host}{$id}{relay} = $relay;
		}
		if (!exists $TO{$host}{$id}{date}) {
			push(@{$TO{$host}{$id}{date}},  $time_st);
			push(@{$TO{$host}{$id}{to}}, $to);
		}

	# POSTFIX milter reject message
	} elsif ($str =~ m#^([^:\s]+): milter-reject: END-OF-MESSAGE from ([^\s]+) ([^;]+); from=<([^>]*)>[,]* to=<([^>]+)>#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		my $relay = &clean_relay(lc($2));
		my $rule = 'milter-reject';
		my $status = $3;
		my $from = &edecode($4);
		my $to = &edecode($5);
		return if ($CONFIG{EXCLUDE_TO} && ($to =~ /^$CONFIG{EXCLUDE_TO}$/));
		if ($status =~ /Virus detected .*\((.*)\)/) {
			$VIRUS{$host}{$id}{file} = 'Inline';
			$VIRUS{$host}{$id}{virus} = $1;
			$VIRUS{$host}{$id}{relay} = $relay;
			$VIRUS{$host}{$id}{from} = $from;
			$VIRUS{$host}{$id}{to} = $to;
			$VIRUS{$host}{$id}{date} =  $time_st;
			if (!exists $FROM{$host}{$id}{date}) {
				$FROM{$host}{$id}{date} =  $time_st;
				$FROM{$host}{$id}{from} = $from;
				$FROM{$host}{$id}{size} = 0;
				$FROM{$host}{$id}{nrcpts} = 1;
				$FROM{$host}{$id}{relay} = $relay;
			}
			if (!exists $TO{$host}{$id}{date}) {
				push(@{$TO{$host}{$id}{date}},  $time_st);
				push(@{$TO{$host}{$id}{to}}, $to);
				$status =~ s/ \(.*//;
				#push(@{$TO{$host}{$id}{status}}, &clear_status($status));
			}
		} else {
			$REJECT{$host}{$id}{relay} = $relay;
			$REJECT{$host}{$id}{date} =  $time_st;
			$REJECT{$host}{$id}{arg1} = &edecode($from);
			$REJECT{$host}{$id}{to} = &edecode($to);
			$REJECT{$host}{$id}{rule} = $rule;
			if ($status =~ /(Rejected due to SPF policy)/i) {
				$REJECT{$host}{$id}{rule} = 'spf-milter';
				$REJECT{$host}{$id}{status} = $1;
			} elsif ($status =~ /(Rejected due to Sender-ID policy)/i) {
				$REJECT{$host}{$id}{rule} = 'sid-milter';
				$REJECT{$host}{$id}{status} = $1;
			} else {
				$REJECT{$host}{$id}{status} = &clear_status($status);
			}
		}
	# POSTFIX some error messages
	} elsif ($str =~ m#^([^:\s]+): ([^=]+)\.\.\. (.*)#) {
		my $id = $KEEP_TEMPORARY{$1} || $1;
		my $to = $2;
		my $status = &clear_status($3);
		$SYSERR{$host}{$id}{date} =  $time_st;
		$SYSERR{$host}{$id}{message} = $status . ': ' . $to;
	# Skip lost connection after STARTTLS duplicate error
	} elsif ($str =~ m#^SSL_accept error from#) {
		return;
	# Catch other messages with sendmail id
	} elsif ($str =~ m#^([^:\s]+): (.*)#) {
		my $id = $1;
		my $err = $2 || '';
		return if (length($id) != 14); # Skip debug lines
		return if ($err =~ /clone|owner/); # Skip mailling list clone
		return if ($err =~ /^(addr|Milter) /); # Skeep milter information
		# Do not store if we already have it
		return if (exists $SYSERR{$host}{$id} || exists $SPAM{$host}{$id} || exists $REJECT{$host}{$id} || exists $TO{$host}{$id}{status});
		my $status = &clear_status($err);
		return if ($status !~ /\s/); # on single word error abort
		$SYSERR{$host}{$id}{date} =  $time_st;
		$SYSERR{$host}{$id}{message} = &clear_status($status);
	# Catch SMTP AUTH
	} elsif ($str =~ m#AUTH=([^,]+), relay=([^,]+), authid=([^,]+), mech=([^,]+), bits=#) {
		my $authid = $3;
		push(@{$AUTH{$host}{$authid}{type}}, $1);
		push(@{$AUTH{$host}{$authid}{mech}}, $4);
		push(@{$AUTH{$host}{$authid}{date}},  $time_st);
		push(@{$AUTH{$host}{$authid}{relay}}, &clean_relay(lc($2)));
	# Catch Anonymous TLS connections
	} elsif ($str =~ m#Anonymous TLS connection established from ([^:]+): (.*) with cipher (.*)#) {
		my $authid = 'anonymous';
		push(@{$AUTH{$host}{$authid}{type}}, $2);
		push(@{$AUTH{$host}{$authid}{mech}}, $3);
		push(@{$AUTH{$host}{$authid}{date}},  $time_st);
		push(@{$AUTH{$host}{$authid}{relay}}, &clean_relay(lc($1)));
	# Catch server TLS connections
	} elsif ($str =~ m#(STARTTLS=[^,]+), relay=([^,]+), version=([^,]+), (verify=[^,]+), cipher=([^,]+), bits=([^,\s]+)#) {
		my $verify = $4;
		push(@{$OTHER{$host}{$time_st}}, "$1, $verify");
		$verify =~ /verify=(.*)/;
		$STARTTLS{$host}{$time_st}{$1}++;
	} elsif ($str =~ m#(STARTTLS=[^,]+), error:(.*), relay=([^,]+)#) {
		push(@{$OTHER{$host}{$time_st}}, "$1, error: $2");
	} else {
		if ($str =~ /(\d{3} \d\.\d\.\d .*)/) {
			$str = $1;
		}
		$str =~ s/.*reject=//;
		push(@{$OTHER{$host}{$time_st}}, &clear_status($str));
	}
}

####
# Parse MailScanner syslog output
####
sub parse_mailscanner
{
	my ($date,$time,$host,$str) = @_;

	return if ($str =~ /is too big for spam checks/);

	my $time_st = "$date$time";

	if ($str =~ /RBL checks: ([^\s]+) found in (.*)/) {
		$SPAM{$host}{$1}{spam} = 'RBL checks';
		$SPAM{$host}{$1}{from} = $FROM{$host}{$1}{from};
		$SPAM{$host}{$1}{to} = $TO{$host}{$1}{queue_to}[0];
		$SPAM{$host}{$1}{relay} = $FROM{$host}{$1}{relay};
		$SPAM{$host}{$1}{date} =  $time_st;
		delete $TO{$host}{$1}{queue_date};
		delete $TO{$host}{$1}{queue_to};
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$1}{date} =  $time_st;
			$SPAMDETAIL{$host}{$1}{type} = 'dnsbl';
			$SPAMDETAIL{$host}{$1}{spam} = $2;
		}
	} elsif ($str =~ /Message ([^\s]+) from (.*) to (.*) is (?:polluriel|spam), ([^\(]+) \(([^\)]*)\)?/) {
		my $id= $1;
		next if ($SPAM{$host}{$id});
		$SPAM{$host}{$id}{from} = $FROM{$host}{$id}{from} || $2;
		$SPAM{$host}{$id}{to} = $TO{$host}{$id}{queue_to}[0] || &edecode($3);
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		$SPAM{$host}{$id}{spam} = 'SpamAssassin';
		$SPAM{$host}{$id}{date} =  $time_st;
		my $text = $5 || '';
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$id}{date} =  $time_st;
			$SPAMDETAIL{$host}{$id}{type} = 'spamassassin';
			$SPAMDETAIL{$host}{$id}{spam} = $text;
		}
		if ($SPAM{$host}{$id}{from} =~ /([a-fA-F0-9\.\:]+) \((.*)\)/) {
			$SPAM{$host}{$id}{relay} = &clean_relay(lc($2));
			$SPAM{$host}{$id}{from} = $1;
		}
		$SPAM{$host}{$id}{from} = &edecode($SPAM{$host}{$id}{from});
		if ($CONFIG{SPAM_DETAIL}) {
			if ($text =~ /(.*), score=(.*), requis [^,]+, (.*)/) {
					$SPAMDETAIL{$host}{$id}{cache} = $1;
					$SPAMDETAIL{$host}{$id}{score} = $2;
					$text = $3;
					if ($text =~ s/autolearn=([^,]+), //) {
						$SPAMDETAIL{$host}{$id}{autolearn} = $1;
					}
					$SPAMDETAIL{$host}{$id}{spam} = $text;
			}
		}
	} elsif ($str =~ /Message ([^\s]+) from (.*?) to (.*?) is (.*)/) {
		my $id= $1;
		next if ($SPAM{$host}{$id});
		$SPAM{$host}{$id}{from} = $FROM{$host}{$id}{from} || &edecode($2);
		$SPAM{$host}{$id}{to} = $TO{$host}{$id}{queue_to}[0] || &edecode($3);
		delete $TO{$host}{$id}{queue_date};
		delete $TO{$host}{$id}{queue_to};
		$SPAM{$host}{$id}{spam} = 'RBL checks';
		$SPAM{$host}{$id}{date} =  $time_st;
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$id}{date} =  $time_st;
			$SPAMDETAIL{$host}{$id}{type} = 'dnsbl';
			$SPAMDETAIL{$host}{$id}{spam} = $4;
		}
		if ($SPAM{$host}{$id}{from} =~ /([a-fA-F0-9\.\:]+) \((.*)\)/) {
			$SPAM{$host}{$id}{relay} = &clean_relay(lc($2));
			$SPAM{$host}{$id}{from} = $1;
		}
		$SPAM{$host}{$id}{from} = &edecode($SPAM{$host}{$id}{from});
	} elsif ($str =~ /Infected message ([^\s]+) from (.*)/) {
		$VIRUS{$host}{$1}{from} = $2;
		$VIRUS{$host}{$1}{date} =  $time_st;
	} elsif ($str =~ /\.\/([^\.]+)\.message: ([^\s]+) FOUND/) {
		$VIRUS{$host}{$1}{file} = 'message';
		$VIRUS{$host}{$1}{virus} = $2;
		$VIRUS{$host}{$1}{date} =  $time_st;
	}
}

####
# Parse Amavis syslog output
####
sub parse_amavis
{
	my ($date, $time ,$host,$str) = @_;

	my $timest = "$date$time";

	my $id = '';
	if ($str =~ /\(([^\)]+)\) (Passed|Blocked) SPAM(.*?), ([^\s]+) [<]*([^\s>]*)[>]* -> ([^\s]+).*, Message-ID: [<]*([^\s>]*)[>]*,.*, Hits: ([^,]+), size: (\d+), queued_as: ([^,]+), (\d+) ms/) {
		my $pid = $1;
		my $status = $2;
		my $relay = lc($4);
		my $msgid = $7;
		my $hits = $8;
		my $size = $9;
		$id = $10;
		my $time = $11;
		my $sender = &edecode($5);
		my $to = &edecode($6);

		if (exists $MSGID{$msgid}) {
			$id = $MSGID{$msgid}{id};
			delete $MSGID{$msgid};
		}
		$SPAM{$host}{$id}{from} = $sender;
		$SPAM{$host}{$id}{to} = $to;
		$SPAM{$host}{$id}{spam} = "Amavis $status Spam";
		$SPAM{$host}{$id}{date} = $timest;
		if (!exists $FROM{$host}{$id}{from}) {
			$FROM{$host}{$id}{from} = $sender;
			$FROM{$host}{$id}{date} = $timest;
			$FROM{$host}{$id}{size} = $size;
			$FROM{$host}{$id}{nrcpts} = 1;
		}
		if (!exists $FROM{$host}{$id}{relay}) {
			$FROM{$host}{$id}{relay} = &clean_relay($relay);
		}

		if (!exists $TO{$host}{$id}{queue_to}) {
			push(@{$TO{$host}{$id}{queue_date}}, $timest);
			push(@{$TO{$host}{$id}{queue_to}}, $to);
		}
		if ($CONFIG{SPAM_DETAIL}) {
			if (!exists $SPAMDETAIL{$host}{$id}) {
				foreach (keys %{$SPAM{$host}{$id}}) {
					$SPAMDETAIL{$host}{$id}{$_} = $SPAM{$host}{$id}{$_} if ($_ ne "spam");
				}
			}
			$SPAMDETAIL{$host}{$id}{type} = 'amavis';
			$SPAMDETAIL{$host}{$id}{score} = $hits;
			$SPAMDETAIL{$host}{$id}{date} = $timest;
			$SPAMDETAIL{$host}{$id}{spam} = $SPAM{$host}{$id}{spam};
		}

	} elsif ($str =~ /\(([^\)]+)\) (Passed|Blocked) SPAM(.*?) [<]*([^\s>]*)[>]* -> [<]*([^,>]*)[>]*,(.*) Message-ID: [<]*([^,>]+)[>]*, (.*)/) {

		my $pid = $1;
		my $status = $2;
		my $relay = lc($3);
		$id = $7;
		my $queueid = $6;
		my $sender = &edecode($4);
		my $to = &edecode($5);
		my $other = $8;
		if ($queueid =~ /Queue-ID: ([^,]+)/) {
			$id = $1;
		} elsif ($other =~ /queued_as: ([^,]+)/) {
			$id = $1;
		} elsif ($str =~ /mail_id: ([^,]+)/) {
			# Quarantine id
			$id = $1;
		}
		$SPAM{$host}{$id}{from} = $sender;
		$SPAM{$host}{$id}{to} = $to;
		$SPAM{$host}{$id}{spam} = "Amavis $status Spam";
		$SPAM{$host}{$id}{date} = $timest;
		if (!exists $FROM{$host}{$id}{from}) {
			$FROM{$host}{$id}{from} = $sender;
			$FROM{$host}{$id}{date} = $timest;
			if ($str =~ /size: (\d+)/) {
				$FROM{$host}{$id}{size} = $1;
			}
			$FROM{$host}{$id}{nrcpts} = 1;
		}
		if (!exists $FROM{$host}{$id}{relay}) {
			$FROM{$host}{$id}{relay} = &clean_relay($relay);
		}
		if (!exists $TO{$host}{$id}{queue_to}) {
			push(@{$TO{$host}{$id}{queue_date}}, $timest);
			push(@{$TO{$host}{$id}{queue_to}}, $to);
		}
		if ($CONFIG{SPAM_DETAIL}) {
			if (!exists $SPAMDETAIL{$host}{$pid}) {
				foreach (keys %{$SPAM{$host}{$id}}) {
					$SPAMDETAIL{$host}{$pid}{$_} = $SPAM{$host}{$id}{$_} if ($_ ne "spam");
				}
			}
			$SPAMDETAIL{$host}{$id}{type} = 'amavis';
			$SPAMDETAIL{$host}{$id}{date} = $timest;
			$SPAMDETAIL{$host}{$id}{spam} = $SPAM{$host}{$id}{spam};
			if ($str =~ / Hits: ([\d\.]+)/) {
				$SPAMDETAIL{$host}{$id}{score} = $1;
			}
		}

	} elsif ($str =~ /\(([^\)]+)\) (Passed|Blocked) SPAM(.*?) [<]*([^\s>]*)[>]* -> [<]*([^,>]*)[>]*, Hits: ([^,]+), (.*)/) {

		my $id = $1;
		my $status = $2;
		my $relay = lc($3);
		my $sender = &edecode($4);
		my $to = &edecode($5);
		my $score = $6;
		my $other = $7;
		if (exists $AMAVIS_ID{$id}) {
			$id = $AMAVIS_ID{$id};
			delete $AMAVIS_ID{$id};
		}
		$SPAM{$host}{$id}{from} = $sender;
			delete $AMAVIS_ID{$id};
		$SPAM{$host}{$id}{to} = $to;
		$SPAM{$host}{$id}{spam} = "Amavis $status Spam";
		$SPAM{$host}{$id}{date} = $timest;
		if (exists $FROM{$host}{$id}{from}) {
			$SPAM{$host}{$id}{from} = $FROM{$host}{$id}{from};
			$SPAM{$host}{$id}{relay} = $FROM{$host}{$id}{relay};
			$SPAM{$host}{$id}{size} = $FROM{$host}{$id}{size};
		}
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$id}{type} = 'amavis';
			$SPAMDETAIL{$host}{$id}{date} = $timest;
			$SPAMDETAIL{$host}{$id}{spam} = $SPAM{$host}{$id}{spam};
			$SPAMDETAIL{$host}{$id}{score}= $score if ($score ne '-');
		}

	} elsif ($str =~ /(Passed|Blocked) INFECTED \(([^\)]*)\)[^,]*, (.*) [<]*([^\s>]*)[>]* -> [<]*([^,>]*)[>]*,(.*) Message-ID: [<]*([^,>]+)[>]*, /) {
		my $virus = $2;
		my $relay = lc($3);
		my $from = $4;
		my $to = &edecode($5);
		$id = &edecode($7);
		my $queue_id = $6;
		if ($queue_id =~ /Queue-ID: ([^,]+),/) {
			$id = $1;
		}

		$VIRUS{$host}{$id}{file} = 'Inline';
		$VIRUS{$host}{$id}{virus} = $virus;
		$VIRUS{$host}{$id}{from} = $from;
		$VIRUS{$host}{$id}{to} = $to;
		$VIRUS{$host}{$id}{date} = $timest;
		if (!exists $FROM{$host}{$id}{from}) {
			$FROM{$host}{$id}{from} = $from;
			$FROM{$host}{$id}{date} = $timest;
			if ($str =~ /size: (\d+)/) {
				$FROM{$host}{$id}{size} = $1;
			}
			$FROM{$host}{$id}{nrcpts} = 1;
		}
		if (!exists $FROM{$host}{$id}{relay}) {
			$FROM{$host}{$id}{relay} = &clean_relay($relay);
		}
		if (!exists $TO{$host}{$id}{queue_to}) {
			push(@{$TO{$host}{$id}{queue_date}}, $timest);
			push(@{$TO{$host}{$id}{queue_to}}, $to);
		}
	}

	if ($CONFIG{SPAM_DETAIL}) {

		if ($str =~ /\(([^\)]+)\) SPAM, (.*), Yes, score=([^\s]+) .* tests=(.*) autolearn=([^,]+)/) {
			my $oid = $1;
			my $from_to = $2;
			my $score = $3;
			my $spam = $4;
			my $autolearn = $5;
			if ($str =~ /autolearn=spam, quarantine ([^\s,]+)/) {
				$id ||= $1;
			}
			$id ||= $oid;
			$SPAMDETAIL{$host}{$id}{date} = $timest;
			$SPAMDETAIL{$host}{$id}{type} = 'amavis';
			$SPAMDETAIL{$host}{$id}{score} = $score;
			$SPAMDETAIL{$host}{$id}{spam} = $spam;
			$SPAMDETAIL{$host}{$id}{autolearn} = $autolearn;
			($SPAMDETAIL{$host}{$id}{from}, $SPAMDETAIL{$host}{$id}{to}) = split(/ -> /, $from_to);
		} elsif ($str =~ /\(([^\)]+)\) SPAM, (.*), Yes, score=([^\s]+).* tests=(.*)/) {
			$id ||= $1;
			my $from_to = $2;
			$SPAMDETAIL{$host}{$id}{date} = $timest;
			$SPAMDETAIL{$host}{$id}{type} = 'amavis';
			$SPAMDETAIL{$host}{$id}{score} = $3;
			$SPAMDETAIL{$host}{$id}{spam} = $4;
			($SPAMDETAIL{$host}{$id}{from}, $SPAMDETAIL{$host}{$id}{to}) = split(/ -> /, $from_to);
		} elsif ($str =~ /\(([^\)]+)\) spam_scan: score=([^\s]+) autolearn=([^\s]+) tests=(.*),/) {
			$id ||= $1;
			$SPAMDETAIL{$host}{$id}{date} = $timest;
			$SPAMDETAIL{$host}{$id}{type} = 'amavis';
			$SPAMDETAIL{$host}{$id}{score} = $2;
			$SPAMDETAIL{$host}{$id}{autolearn} = $3;
			$SPAMDETAIL{$host}{$id}{spam} = $4;
		} elsif ($str =~ /\(([^\)]+)\) SPAM, (.*), Yes, hits=([^\s]+) .*tests=(.*), quarantine/) {
			$id ||= $1;
			my $from_to = $2;
			$SPAMDETAIL{$host}{$id}{date} = $timest;
			$SPAMDETAIL{$host}{$id}{type} = 'amavis';
			$SPAMDETAIL{$host}{$id}{score} = $3;
			$SPAMDETAIL{$host}{$id}{spam} = $4;
			($SPAMDETAIL{$host}{$id}{from}, $SPAMDETAIL{$host}{$id}{to}) = split(/ -> /, $from_to);
		}
	}
}

####
# Parse clamsmtpd syslog output
####
sub parse_clamsmtpd
{
	my ($date, $time ,$host,$str) = @_;

	my $timest = "$date$time";

	if ($str =~ /^([^:]+): from=([^,]+), to=([^,]+), status=VIRUS:(.*)/) {
		my $virus = $4;
		my $from = &edecode($2);
		my $to = &edecode($3);
		my $id = $1;

		foreach my $i (keys %{$FROM{$host}}) {
			if ($FROM{$host}{$i}{from} = $from) {
				$id = $i;
				last;
			}
		}
		$VIRUS{$host}{$id}{file} = 'Inline';
		$VIRUS{$host}{$id}{virus} = $virus;
		$VIRUS{$host}{$id}{from} = $from;
		$VIRUS{$host}{$id}{to} = $to;
		$VIRUS{$host}{$id}{date} = $timest;
	}
}


####
# Parse MimeDefang syslog output
####
sub parse_mimedefang
{
	my ($date,$time,$host,$str) = @_;

	my $time_st = "$date$time";

	#### Store each relevant information per date and ids
	#MDLOG,sendmail_queue_id,spam,score,relay,<from@sender>,<to@sdest>,subject
	#MDLOG,sendmail_queue_id,virus,virus_name,relay,<from@sender>,<to@sdest>,subject
	if ($str =~ /MDLOG,([^,]+),spam,([^,]+),([^,]+),([^,]+),([^,]+),(.*)/) {
		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$1}{type} = 'mimedefang';
			$SPAMDETAIL{$host}{$1}{score} = $2;
			$SPAMDETAIL{$host}{$1}{spam} = $6;
			$SPAMDETAIL{$host}{$1}{date} =  $time_st;
		}
		$SPAM{$host}{$1}{spam} = 'mimedefang';
		$SPAM{$host}{$1}{date} =  $time_st;
		$SPAM{$host}{$1}{relay} = &clean_relay(lc($3));
		$SPAM{$host}{$1}{from} = $FROM{$host}{$1}{from} || &edecode($4);
		$SPAM{$host}{$1}{to} = $TO{$host}{$1}{queue_to}[0] || &edecode($5);
		delete $TO{$host}{$1}{queue_date};
		delete $TO{$host}{$1}{queue_to};
	} elsif ($str =~ /MDLOG,([^,]+),virus,([^,]+),([^,]+),/) {
		$VIRUS{$host}{$1}{virus} = $2;
		$VIRUS{$host}{$1}{file} = 'Inline';
		$VIRUS{$host}{$1}{date} =  $time_st;
		$VIRUS{$host}{$1}{relay} = &clean_relay(lc($3));
	}
}

####
# Parse Postgrey syslog output
####
sub parse_postgrey
{
	my ($date,$time,$host,$str) = @_;

	my $time_st = "$date$time";

	if ($str =~ /action=([^,]+), reason=([^,]+), client_name=([^,]+), client_address=([^,]+), sender=([^,]+), recipient=(.*)/) {

		my $action = $1;
		my $status = $2;
		my $relay_name = $3;
		my $relay_ip = $4;
		my $sender = &edecode($5);
		my $to = &edecode($6);
		my $id = &get_uniqueid();
		$status =~ s/ \(.*//;
		$POSTGREY{$host}{$id}{action} = $action;
		$POSTGREY{$host}{$id}{relay} = $relay_name || $relay_ip;
		$POSTGREY{$host}{$id}{from} = &edecode($sender);
		$POSTGREY{$host}{$id}{to} = &edecode($to);
		$POSTGREY{$host}{$id}{status} = $status;
		$POSTGREY{$host}{$id}{date} =  $time_st;

	} elsif ($str =~ s/^grey:\s+//) {

		&parse_sqlgrey($date,$time_st,$host,$str);

	} elsif ($str =~ s/^spam:\s+//) {

		&parse_sqlgrey_spam($date,$time_st,$host,$str);
	}
	

}

####
# Parse sqlgrey spam syslog output
####
sub parse_sqlgrey_spam
{
	my ($date,$time_st,$host,$str) = @_;

	if ($str =~ /^(.*): (.*) -> ([^\s]+) /) {
		my $relay = lc($1);
		my $sender = &edecode($2);
		my $to = &edecode($3);
		my $id = &get_uniqueid();
		$SPAM{$host}{$id}{relay} = $relay;
		$SPAM{$host}{$id}{from} = $sender;
		$SPAM{$host}{$id}{to} = $to;
		$SPAM{$host}{$id}{spam} = "sqlgrey spam";
		$SPAM{$host}{$id}{date} = $time_st;

		$FROM{$host}{$id}{from} = $sender;
		$FROM{$host}{$id}{date} = $time_st;
		$FROM{$host}{$id}{nrcpts} = 1;
		$FROM{$host}{$id}{relay} = $relay;
	}

}

####
# Parse sqlgrey syslog output
####
sub parse_sqlgrey
{
	my ($date,$time_st,$host,$str) = @_;


	my $id = &get_uniqueid();
	if ($str =~ /^domain awl match: updating ([^,]+), (.*)/) {
		$POSTGREY{$host}{$id}{status} = 'passed domain awl match';
		$POSTGREY{$host}{$id}{action} = 'passed';
		$POSTGREY{$host}{$id}{relay} = &clean_relay($1);
		$POSTGREY{$host}{$id}{from} = 'unset@'. &edecode($2);
		$POSTGREY{$host}{$id}{date} =  $time_st;
	} elsif ($str =~ /^from awl match: updating ([^,]+), ([^\(]+)/) {
		$POSTGREY{$host}{$id}{status} = 'passed from awl match';
		$POSTGREY{$host}{$id}{action} = 'passed';
		$POSTGREY{$host}{$id}{relay} = $1;
		$POSTGREY{$host}{$id}{from} = &edecode($2);
		$POSTGREY{$host}{$id}{date} =  $time_st;
	} elsif ($str =~ /^early reconnect: ([^,]+), (.*) -> (.*)/) {
		$POSTGREY{$host}{$id}{status} = 'early reconnect';
		$POSTGREY{$host}{$id}{action} = 'early';
		$POSTGREY{$host}{$id}{relay} = $1;
		$POSTGREY{$host}{$id}{from} = &edecode($2);
		$POSTGREY{$host}{$id}{to} = &edecode($3);
		$POSTGREY{$host}{$id}{date} =  $time_st;
	} elsif ($str =~ /^reconnect ok: ([^,]+), (.*) -> (.*) \((.*)\)/) {
		$POSTGREY{$host}{$id}{status} = 'passed reconnect ok';
		$POSTGREY{$host}{$id}{action} = 'passed';
		$POSTGREY{$host}{$id}{relay} = $1;
		$POSTGREY{$host}{$id}{from} = &edecode($2);
		$POSTGREY{$host}{$id}{to} = &edecode($3);
		$POSTGREY{$host}{$id}{date} =  $time_st;
	} elsif ($str =~ /^new: ([^,]+), (.*) -> (.*)/) {
		$POSTGREY{$host}{$id}{status} = 'new delayed';
		$POSTGREY{$host}{$id}{action} = 'delayed';
		$POSTGREY{$host}{$id}{relay} = $1;
		$POSTGREY{$host}{$id}{from} = &edecode($2);
		$POSTGREY{$host}{$id}{to} = &edecode($3);
		$POSTGREY{$host}{$id}{date} =  $time_st;
	}
}

####
# Parse spamd syslog output
####
sub parse_spamd
{
	my ($date,$time,$host,$str) = @_;

	my $time_st = "$date$time";

	#### Store each relevant information per date and ids
	if ($str =~ /result: Y ([^\s]+) - (.*) scantime=.*mid=<(.*)>,.*autolearn=(.*)/) {

		my $score = $1;
		my $spam = $2;
		my $msgid = $3;
		my $autolearn = $4;

		my $id = &get_uniqueid();

		$SPAM{$host}{$id}{spam} = 'spamd';
		$SPAM{$host}{$id}{date} =  $time_st;
		$SPAM{$host}{$id}{mid} = $msgid;

		if ($CONFIG{SPAM_DETAIL}) {
			$SPAMDETAIL{$host}{$id}{type} = 'spamdmilter';
			$SPAMDETAIL{$host}{$id}{score} = $score;
			$SPAMDETAIL{$host}{$id}{spam} = $spam;
			$SPAMDETAIL{$host}{$id}{date} =  $time_st;
			$SPAMDETAIL{$host}{$id}{mid} = $msgid;
			$SPAMDETAIL{$host}{$id}{autolearn} = $autolearn;
		}

		foreach my $mid (keys %{$FROM{$host}}) {

			next if (!exists $FROM{$host}{$mid}{msgid});

			# Some message id can be truncated in from log and full in spamd message
			if ($SPAM{$host}{$id}{mid} =~ /^\Q$FROM{$host}{$mid}{msgid}\E/) {

				$SPAM{$host}{$mid}{from} = $FROM{$host}{$mid}{sender};
				$SPAM{$host}{$mid}{spam} = $SPAM{$host}{$id}{spam};
				$SPAM{$host}{$mid}{date} = $SPAM{$host}{$id}{date};
				$SPAM{$host}{$mid}{mid}  = $SPAM{$host}{$id}{mid};
				$FROM{$host}{$mid}{msgid} = $SPAM{$host}{$id}{mid};
				delete $SPAM{$host}{$id};
				if ($CONFIG{SPAM_DETAIL}) {
					$SPAMDETAIL{$host}{$mid}{type} = 'spamdmilter';
					$SPAMDETAIL{$host}{$mid}{score} = $SPAMDETAIL{$host}{$id}{score};
					$SPAMDETAIL{$host}{$mid}{spam} = $SPAMDETAIL{$host}{$id}{spam};
					$SPAMDETAIL{$host}{$mid}{date} = $SPAMDETAIL{$host}{$id}{date};
					$SPAMDETAIL{$host}{$mid}{mid} = $mid;
					$SPAMDETAIL{$host}{$mid}{autolearn} = $SPAMDETAIL{$host}{$id}{autolearn};
					delete $SPAMDETAIL{$host}{$id};
				}
				last;
			}
		}
	}
}

####
# Decode email address and keep only email part
####
sub edecode
{
	my ($addr) = @_;

	if ($addr =~ /=\?[^\?]+\?(.)\?(.*)?=/s) {
		if (uc($1) eq 'B') {
			$addr = decode_base64($1);
		} elsif (uc($1) eq 'Q') {
			$addr = decode_qp($1);
		}
	}
	$addr =~ s#^\s+##;
	$addr =~ s#\s+$##;
	$addr =~ s#[<>]##g;
	$addr =~ s#,$##;
	$addr =~ s#:##g;
	$addr =~ s#'##g;
	$addr =~ s# \(\d+/\d+\)##g;
	if ($addr !~ /\@/) {
		$addr .= $CONFIG{DEFAULT_DOMAIN} || '@localhost';
	}

	return lc($addr);
}

####
# Decode subject
####
sub decode_subject
{
	my ($str) = @_;

	while ($str =~ /=\?[^\?]+\?(.)\?(.*?)\?=/) {
		my $subject = $1;
		if (uc($subject) eq 'B') {
			$subject = decode_base64($2);
		} elsif (uc($subject) eq 'Q') {
			$subject = decode_qp($2);
		}
		$str =~ s/=\?[^\?]+\?(.)\?(.*?)\?=/$subject/;
	}

	while ($str =~ /=\?[^\?]+\?(.)\?(.*?)/) {
		my $subject = $1;
		if (uc($subject) eq 'B') {
			$subject = decode_base64($2);
		} elsif (uc($subject) eq 'Q') {
			$subject = decode_qp($2);
		}
		$str =~ s/=\?[^\?]+\?(.)\?(.*?)/$subject/;
	}
	$str =~ s/\?\?//g;
	return $str;
}


####
# Clean relay address
####
sub clean_relay
{
	my ($relay) = @_;

	if ($relay =~ m#\b([a-fA-F0-9\.\:]+) \(may be forged#) {
		return $1;
	} elsif ($relay =~ m#localhost|127\.0\.0\.1#) {
		return 'localhost';
	} elsif ( $relay =~ s/\[([^\]]+)\]// ) {
		my $fqdn = $relay;
		my $ip = $1;
		$fqdn =~ s#:.*##;
		if (!$fqdn || ($fqdn eq 'unknown')) {
			return $ip;
		} elsif ($fqdn =~ /[\s,]/) {
			return $ip;
		} else {
			return $fqdn;
		}
	} elsif ( $relay =~ s/\(([a-fA-F0-9\.\:]+)\)// ) {
		return $1;
	}
	$relay =~ s#^\s+##;
	$relay =~ s#\s+.*##;
	$relay =~ s#\.$##;
	$relay =~ s#\s.*##;

	return $relay;
}

####
# Set script internal date/time format from localtime call
# Format: YYYYMMDDHHMMSS
####
sub format_time
{
	my ($sec,$min,$hour,$mday,$mon,$year) = @_;

	$hour = sprintf("%02d", $hour);
	$min  = sprintf("%02d", $min);
	$sec  = sprintf("%02d", $sec);

	return 1900+$year . sprintf("%02d", $mon+1) . sprintf("%02d", $mday) . "$hour$min$sec";
}

####
# Flush memory stored data to disk
####
sub flush_data
{
	my $final = shift;

	# In incremental mode if there's still no line parsed get out of there
	return if ($OLD_LAST_PARSED ne '');

	####
	# Data are saved on disk as follow:
	# 	$host/$year/$month/$day/filename.dat
	####
	
	# Init greylisting temporary storage
	%GREYLIST = ();
	my %EXCLUDED = ();

	# Save senders informations first
	&dprint("Writing sender data to disk...");
	my $nobj = 0;
	foreach my $host (keys %FROM) {
		my %senders = ();
		foreach my $id (keys %{$FROM{$host}}) {

			if (exists $POSTFIX_PLUGIN_TEMP_DELIVERY{$id}) {
				delete $FROM{$host}{$id};
				next;
			}
			# Check from sender or sender relay exclusion
			if ($CONFIG{EXCLUDE_FROM} && ($FROM{$host}{$id}{from} =~ /^$CONFIG{EXCLUDE_FROM}$/)) {
				$EXCLUDED{$id} = 1;
				delete $FROM{$host}{$id};
				next;
			}
			if ($CONFIG{EXCLUDE_RELAY} && ($FROM{$host}{$id}{relay} =~ /^$CONFIG{EXCLUDE_RELAY}$/)) {
				$EXCLUDED{$id} = 1;
				delete $FROM{$host}{$id};
				next;
			}
			foreach (keys %MSGID) {
				delete $MSGID{$_} if ($MSGID{$_}{id} eq $id);
			}
			delete $SKIPMSG{$id};

			next if ($FROM{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
			# Key: year/month/day, Format: Hour:Id:Sender:Size:Nrcpts:Relay:Subject
			$senders{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $FROM{$host}{$id}{from} . ':' . $FROM{$host}{$id}{size} . ':' . $FROM{$host}{$id}{nrcpts} . ':' . $FROM{$host}{$id}{relay} . ':' . $FROM{$host}{$id}{subject} ."\n";
			$nobj++;
			# Complete Spam information
			if (exists $SPAM{$host}{$id}) {
				$SPAM{$host}{$id}{date} = $FROM{$host}{$id}{date};
				$SPAM{$host}{$id}{from} = $FROM{$host}{$id}{from};
			}

			# Find real sendmail id for Amavis virus message
			my $msgid = $FROM{$host}{$id}{msgid} || '';
			if ($msgid && exists($VIRUS{$host}{$msgid})) {
				foreach my $k (keys %{$VIRUS{$host}{$msgid}}) {
					$VIRUS{$host}{$id}{$k} = $VIRUS{$host}{$msgid}{$k};
				}
				delete $VIRUS{$host}{$msgid};
			}
			# Find real sendmail id for Amavis spam message
			if ($msgid && exists($SPAM{$host}{$msgid})) {
				foreach my $k (keys %{$SPAM{$host}{$msgid}}) {
					$SPAM{$host}{$id}{$k} = $SPAM{$host}{$msgid}{$k};
				}
				delete $SPAM{$host}{$msgid};
			}
			if ($msgid && exists($SPAMDETAIL{$host}{$msgid})) {
				foreach my $k (keys %{$SPAMDETAIL{$host}{$msgid}}) {
					$SPAMDETAIL{$host}{$id}{$k} = $SPAMDETAIL{$host}{$msgid}{$k};
				}
				delete $SPAMDETAIL{$host}{$msgid};
			}
		}
		delete $FROM{$host};
		foreach my $dir (keys %senders) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/senders.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/senders.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $senders{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj sender objects");
	# clear all senders memory storage
	%FROM = ();
	
	&dprint("Writing reject data to disk...");
	$nobj = 0;
	# Save rejected messages
	foreach my $host (keys %REJECT) {
		my %rejected = ();
		foreach my $id (keys %{$REJECT{$host}}) {
			if (exists $EXCLUDED{$id}) {
				delete $REJECT{$host}{$id};
				next;
			}
			$REJECT{$host}{$id}{date} =~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/;
			# Key: year/month/day, Format: Hour:Id:Rule:Relay:Arg1:Status
			$rejected{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $REJECT{$host}{$id}{rule} . ':' . $REJECT{$host}{$id}{relay} . ':' . $REJECT{$host}{$id}{arg1} . ':' . $REJECT{$host}{$id}{status} . "\n";
			$nobj++;
		}
		delete $REJECT{$host};
		foreach my $dir (keys %rejected) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/rejected.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/rejected.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $rejected{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj reject object.");
	# clear all senders memory storage
	%REJECT = ();
		
	&dprint("Writing DSN data to disk...");
	$nobj = 0;
	# Save DSN messages
	foreach my $host (keys %DSN) {
		my %dsned = ();
		foreach my $id (keys %{$DSN{$host}}) {
			if (exists $EXCLUDED{$id}) {
				delete $DSN{$host}{$id};
				next;
			}
			next if ($DSN{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
			# Key: year/month/day, Format: Hour:Id:SourceId:Status
			$dsned{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $DSN{$host}{$id}{srcid} . ':' . $DSN{$host}{$id}{status} . "\n";
			$nobj++;
		}
		delete $DSN{$host};
		foreach my $dir (keys %dsned) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/dsn.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/dsn.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $dsned{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj DSN object.");
	# clear all senders memory storage
	%DSN = ();
	
	# Save recipients informations
	&dprint("Writing recipient data to disk...");
	$nobj = 0;
	foreach my $host (keys %TO) {
		my %rcpts = ();
		foreach my $id (keys %{$TO{$host}}) {
			if (exists $POSTFIX_PLUGIN_TEMP_DELIVERY{$id}) {
				delete $TO{$host}{$id};
				delete $POSTFIX_PLUGIN_TEMP_DELIVERY{$id};
				next;
			}
			if (exists $EXCLUDED{$id}) {
				delete $TO{$host}{$id};
				next;
			}
			for (my $i = 0; $i <= $#{$TO{$host}{$id}{date}}; $i++) { 
				# Key: year/month/day, Format: Hour:Id:Rcpt:Relay:Status
				next if ($TO{$host}{$id}{date}[$i] !~ /(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
				$rcpts{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $TO{$host}{$id}{to}[$i] . ':' . $TO{$host}{$id}{relay}[$i] . ':' . $TO{$host}{$id}{status}[$i] . "\n";
				$nobj++;
			}
			for (my $i = 0; $i <= $#{$TO{$host}{$id}{queue_date}}; $i++) { 
				# Key: year/month/day, Format: Hour:Id:Rcpt:Relay:Status
				next if ($TO{$host}{$id}{queue_date}[$i] !~ /(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
				$rcpts{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $TO{$host}{$id}{queue_to}[$i] . ":none:Queued\n";
				$nobj++;
			}
			# Complete Spam information
			if (exists $SPAM{$host}{$id} && !exists $SPAM{$host}{$id}{to}) {
				$SPAM{$host}{$id}{to} = $TO{$host}{$id}{to}[0];
			}
		}
		delete $TO{$host};
		foreach my $dir (keys %rcpts) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/recipient.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/recipient.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $rcpts{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj recipient object.");
	# clear all recipients memory storage
	%TO = ();

	# Save Spam objects
	&dprint("Writing Spam data to disk...");
	$nobj = 0;
	foreach my $host (keys %SPAM) {
		my %spams = ();
		foreach my $id (keys %{$SPAM{$host}}) {
			if (exists $EXCLUDED{$id}) {
				delete $SPAM{$host}{$id};
				next;
			}
			next if ($SPAM{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
			# Key: year/month/day, Format: Hour:Id:from:to:spam
			$spams{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $SPAM{$host}{$id}{from} . ':' . $SPAM{$host}{$id}{to} . ':' . $SPAM{$host}{$id}{spam} . "\n";
			$nobj++;
		}
		delete $SPAM{$host};
		foreach my $dir (keys %spams) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/spam.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/spam.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $spams{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj spam object.");
	# clear all spams memory storage
	%SPAM = ();
	%SPAMPD = ();

	# Save Spam detail objects
	&dprint("Writing Spam detail data to disk...");
	$nobj = 0;
	foreach my $host (keys %SPAMDETAIL) {
		my %spamdetails = ();
		foreach my $id (keys %{$SPAMDETAIL{$host}}) {
			if (exists $EXCLUDED{$id}) {
				delete $SPAMDETAIL{$host}{$id};
				next;
			}
			next if (!$SPAMDETAIL{$host}{$id}{type});
			next if ($SPAMDETAIL{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
			# Key: year/month/day, Format: Hour:Id:type:score:cache:autolearn:spam
			$spamdetails{"$1/$2/$3"}{$SPAMDETAIL{$host}{$id}{type}} .= "$4$5$6" . ':' . $id . ':' . $SPAMDETAIL{$host}{$id}{type} . ':' . $SPAMDETAIL{$host}{$id}{score} . ':' . $SPAMDETAIL{$host}{$id}{cache} . ':' . $SPAMDETAIL{$host}{$id}{autolearn} . ':' . $SPAMDETAIL{$host}{$id}{spam} . "\n";
			$nobj++;
		}
		delete $SPAMDETAIL{$host};
		foreach my $dir (keys %spamdetails) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			foreach my $type (keys %{$spamdetails{$dir}}) {
				if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/$type.dat") ) {
					&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/$type.dat: $!");
					&logerror("Data will be lost.");
					next;
				} else {
					&dprint("Writing Spam detail for $type into $CONFIG{OUT_DIR}/$host/$dir/$type.dat");
					print OUT $spamdetails{$dir}{$type};
					close(OUT);
				}
			}
		}
	}
	&dprint("\tWrote $nobj spam detail object.");
	# clear all spams memory storage
	%SPAMDETAIL = ();

	# Save Postgrey objects
	&dprint("Writing Postgrey detail data to disk...");
	$nobj = 0;
	foreach my $host (keys %POSTGREY) {
		my %postgreys = ();
		foreach my $id (keys %{$POSTGREY{$host}}) {
			if (exists $EXCLUDED{$id}) {
				delete $POSTGREY{$host}{$id};
				next;
			}
			next if ($POSTGREY{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
			# Key: year/month/day, Format: Hour:Id:relay:from:to:action:status
			$postgreys{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $POSTGREY{$host}{$id}{relay} . ':' . $POSTGREY{$host}{$id}{from} . ':' . $POSTGREY{$host}{$id}{to} . ':' . $POSTGREY{$host}{$id}{action} . ':' . $POSTGREY{$host}{$id}{status} . "\n";
			$nobj++;
		}
		delete $POSTGREY{$host};
		foreach my $dir (keys %postgreys) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/postgrey.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/postgrey.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $postgreys{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj postgrey object.");
	# clear all postgrey memory storage
	%POSTGREY = ();

	# Save Virus objects
	&dprint("Writing Virus data to disk...");
	$nobj = 0;
	foreach my $host (keys %VIRUS) {
		my %viruses = ();
		foreach my $id (keys %{$VIRUS{$host}}) {
			if (exists $EXCLUDED{$id}) {
				delete $VIRUS{$host}{$id};
				next;
			}
			next if ($VIRUS{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
			# Key: year/month/day, Format: Hour:Id:file:virus
			$viruses{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $VIRUS{$host}{$id}{file} . ':' . $VIRUS{$host}{$id}{virus} . "\n";
			$nobj++;
		}
		delete $VIRUS{$host};
		foreach my $dir (keys %viruses) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/virus.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/virus.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $viruses{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj virus object.");
	# clear all viruses memory storage
	%VIRUS = ();

	# Save syserr objects
	&dprint("Writing syserr data to disk...");
	$nobj = 0;
	foreach my $host (keys %SYSERR) {
		my %errors = ();
		foreach my $id (keys %{$SYSERR{$host}}) {
			if (exists $EXCLUDED{$id}) {
				delete $SYSERR{$host}{$id};
				next;
			}
			next if ($SYSERR{$host}{$id}{date} !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
			# Key: year/month/day, Format: Hour:Id:Error
			$errors{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $SYSERR{$host}{$id}{message} . "\n";
			$nobj++;
		}
		delete $SYSERR{$host};
		foreach my $dir (keys %errors) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/syserr.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/syserr.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $errors{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj syserr object.");
	# clear all syserr memory storage
	%SYSERR = ();


	# Save other message objects
	&dprint("Writing warning message data to disk...");
	$nobj = 0;
	foreach my $host (keys %OTHER) {
		my %errors = ();
		foreach my $dt (keys %{$OTHER{$host}}) {
			next if ($dt !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
			foreach my $v (@{$OTHER{$host}{$dt}}) {
				# Key: year/month/day, Format: Hour:Error
				$errors{"$1/$2/$3"} .= "$4$5$6" . ':' . $v . "\n";
				$nobj++;
			}
		}
		delete $OTHER{$host};
		foreach my $dir (keys %errors) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/other.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/other.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $errors{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj warning message object.");
	# clear all warning message memory storage
	%OTHER = ();

	# Save STARTTLS stats
	&dprint("Writing STARTTLS statistics to disk...");
	$nobj = 0;
	foreach my $host (keys %STARTTLS) {
		my %starttls = ();
		foreach my $dt (keys %{$STARTTLS{$host}}) {
			next if ($dt !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
			# Key: year/month/day, Format: Hour:Id:FAIL=count;NO=count;OK=count;
			$starttls{"$1/$2/$3"} .= "$4$5$6" . ':';
			foreach my $v (keys %{$STARTTLS{$host}{$dt}}) {
				$starttls{"$1/$2/$3"} .= $v . '=' . ($STARTTLS{$host}{$dt}{$v} || 0) . ';';
				$nobj++;
			}
			$starttls{"$1/$2/$3"} .= "\n";
			$starttls{"$1/$2/$3"} =~ s/;$//s;
		}
		delete $STARTTLS{$host};
		foreach my $dir (keys %starttls) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/starttls.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/starttls.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $starttls{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj STARTTLS objects.");
	# clear all warning message memory storage
	%STARTTLS = ();

	# Save auth message objects
	&dprint("Writing warning auth data to disk...");
	$nobj = 0;
	foreach my $host (keys %AUTH) {
		my %authent = ();
		foreach my $id (keys %{$AUTH{$host}}) {
			if (exists $EXCLUDED{$id}) {
				delete $AUTH{$host}{$id};
				next;
			}
			for (my $i = 0; $i <= $#{$AUTH{$host}{$id}{date}}; $i++) { 
				next if ($AUTH{$host}{$id}{date}[$i] !~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/);
				# Key: year/month/day, Format: date:id:relay:mech:type
				$authent{"$1/$2/$3"} .= "$4$5$6" . ':' . $id . ':' . $AUTH{$host}{$id}{relay}[$i] . ':' . $AUTH{$host}{$id}{mech}[$i] . ':' . $AUTH{$host}{$id}{type}[$i] . "\n";
				$nobj++;
			}
		}
		delete $AUTH{$host};
		foreach my $dir (keys %authent) {
			if (!-d "$CONFIG{OUT_DIR}/$host/$dir") {
				&create_directory("$host/$dir");
			}
			if (not open(OUT, ">>$CONFIG{OUT_DIR}/$host/$dir/auth.dat") ) {
				&logerror("Can't write to file $CONFIG{OUT_DIR}/$host/$dir/auth.dat: $!");
				&logerror("Data will be lost.");
				next;
			} else {
				print OUT $authent{$dir};
				close(OUT);
			}
		}
	}
	&dprint("\tWrote $nobj auth object.");
	# clear all auth storage
	%AUTH = ();

	# Write last parsed data
	if (!$CONFIG{FORCE}) {
		if (not open(OUT, ">$CONFIG{OUT_DIR}/$LAST_PARSE_FILE") ) {
			&logerror("Can't write to file $CONFIG{OUT_DIR}/$LAST_PARSE_FILE: $!");
		} else {
			&dprint("Writing last parsed line and last log file reading offset...");
			print OUT "$LAST_PARSED\t$OLD_OFFSET";
			close(OUT);
		}
	}
}

####
# Create output directory tree
####
sub create_directory
{
	my $dest = shift;

	my $curdir = '';
	foreach my $d (split(/\//, $dest)) {
		$curdir .= $d . '/';
		if (!-d "$CONFIG{OUT_DIR}/$curdir") {
			if (not mkdir("$CONFIG{OUT_DIR}/$curdir")) {
				&logerror("Can't create directory $CONFIG{OUT_DIR}/$curdir: $!");
				&logerror("Data will be lost.");
				return 0;
			}
		}
	}

	return 1;
}

####
# Routine used to log sendmailanalyzer errors or send emails alert if requested
####
sub logerror
{
	my $str = shift;
	
	print STDERR "ERROR: $str\n";
	
}

####
# Read configuration file
####
sub read_config
{
	my $file = shift;

	if (!-e $file) {
		$file = '/etc/sendmailanalyzer.conf';
	}
	if (!-e $file) {
		&logerror("Configuration file $file doesn't exists");
		return;
	} else {
		if (not open(IN, $file)) {
			&logerror("Can't read configuration file $file: $!");
		} else {
			while (<IN>) {
				chomp;
				s/#.*//;
				s/^[\s\t]+//;
				s/[\s\t]$//;
				if ($_ ne '') {
					my ($var, $val) = split(/[\s\t]+/, $_, 2); 
					$CONFIG{$var} = $val if (!defined $CONFIG{$var} && ($val ne ''));
				}
			}
			close(IN);
		}
	}
	# Set default values
	$CONFIG{LOG_FILE} ||= '/var/log/maillog';
	$CONFIG{ZCAT_PROG} ||= '/usr/bin/zcat';
	$CONFIG{TAIL_PROG} ||= '/usr/bin/tail';
	$CONFIG{TAIL_ARGS} ||= '-n 0 -f';
	$CONFIG{OUT_DIR} ||= '/var/www/sendmailanalyzer';
	$CONFIG{PID_DIR} ||= $CONFIG{PID_FILE};
	$CONFIG{DEBUG} ||= 0;
	$CONFIG{FULL} ||= 0;
	$CONFIG{FORCE} ||= 0;
	$CONFIG{BREAK} ||= 0;
	$CONFIG{DELAY} ||= 5;
	$CONFIG{MTA_NAME} ||= 'sm-mta|sendmail|postfix|spampd';
	$CONFIG{MAILSCAN_NAME} ||= 'MailScanner';
	$CONFIG{CLAMD_NAME} ||= 'clamd';
	$CONFIG{MD_NAME} ||= 'mimedefang.pl';
	$CONFIG{AMAVIS_NAME} ||= 'amavis';
	if (!exists $CONFIG{SPAM_DETAIL}) {
		$CONFIG{SPAM_DETAIL} = 1;
	}
	$CONFIG{CLAMSMTPD_NAME} = 'clamsmtpd';
	$CONFIG{MTA_NAME} =~ s/[\s\t,;]+/\|/g;
	$CONFIG{PID_DIR} ||= '/var/run';
	$CONFIG{POSTGREY_NAME} ||= 'postgrey|sqlgrey';
	$CONFIG{SPAMD_NAME} ||= 'spamd';
}

####
# Routine to remove any spercific part of status line to report
# generic status
####
sub clear_status
{
	my ($status, $id) = @_;

	# Try to avoid doublon when postfix send the message to a plugin
	# and then the plugin will use postfix again to send the message.
	if ($status =~  /sent \(250 .* ([0-9A-F]+)\)/) {
		$POSTFIX_PLUGIN_TEMP_DELIVERY{$1} = $id;
	}

	# Try to detect Amavis id from messages status
	if ($status =~  /^[^\s]+ \(.*, id=(\d+\-\d+) - (.*)\)/) {
		$AMAVIS_ID{$1} = $id;
	}

	# Anonymize ip addresse in status message
        $status =~ s/\[[a-fA-F0-9\.\:]+\]/.../g;

	$status =~ s/(IP name possibly forged).*/$1/;
	$status =~ s/(IP name lookup failed).*/$1/;

	if ($status =~ /^Sent.*/i) {
		return 'Sent';
	} elsif ($status =~ /Deferred.*/i) {
		return 'Deferred';
	} elsif ($status =~ s/collect: //) {
		$status =~ s/ from.*//;
		return $status;
	} elsif ($status =~ /bounced.*/i) {
		return 'Bounced';
	} elsif ($status =~ /hostname (.*) does not resolve to address ([^:]+): (.*)/) {
		return "hostname does not resolve to address, $3";
	} elsif ($status =~ /hostname (.*) does not resolve to address ([^:]+)/) {
		return "hostname does not resolve to address";
	} elsif ($status =~ /(numeric domain name)/) {
		return "numeric domain name";
	} elsif ($status =~ /(address not listed for hostname)/) {
		return $1;
	} elsif ($status =~ /warning: (Illegal address syntax).* in ([^\s]+ command):/) {
		return "$1 $2";
	} elsif ($status =~ /warning: (non-SMTP command)/) {
		return "$1";
	} elsif ($status =~ /warning: [^:]+:\d+ ([^:]+)/) {
		return $1;
	} elsif ($status =~ /warning: [^:]+: ([^:]+)/) {
		return $1;
	} elsif ($status =~ /did not issue/) {
		$status =~ s/.*did not issue/No/;
		return $status;
	} elsif ($status =~ /(Greylisting in action)/i) {
		return $1;
	} elsif ($status =~ /(lost input channel ).*(after.*)/) {
		$status = $1 . $2;
		return $status;
	} elsif ($status =~ /(timeout waiting for input ).*(during.*)/) {
		$status = $1 . $2;
		return $status;
	} elsif ($status =~ /(timeout writing message).*(: [^:]+?)( by |$)/) {
		$status = $1 . $2;
		return $status;
	} elsif ($status =~ /(timeout writing message)/) {
		return $1;
        } elsif ($status =~ /(rejecting commands ).* (due to pre-greeting traffic)/) {
		return $1 . $2;
        } elsif ($status =~ /(rejecting commands ).*(due to.*)/) {
		$status = $1 . $2;
		return $status;
        } elsif ($status =~ /(readqf: ).*:( .*)/) {
		$status = $1 . $2;
		return $status;
        } elsif ($status =~ /(Syntax error in mailbox address)/) {
		return $1;
        } elsif ($status =~ /(Possible SMTP RCPT flood, throttling)/) {
		return $1;
        } elsif ($status =~ /(Domain name required for sender address)/) {
		return $1;
        } elsif ($status =~ /(STARTTLS=[^,]+),.*(, verify=[^,]+)/) {
		return $1 . $2;
        } elsif ($status =~ /, stat=(.*)/) {
		return $1;
        } elsif ($status =~ /(rejecting connections on daemon[^:]*: [^:]*)/) {
		return $1;
        } elsif ($status =~ /(VRFY ([^\s]+) rejected)/) {
		return "VRFY rejected";
        } elsif ($status =~ /: (sender notify:.*)/) {
		return $1;
        } elsif ($status =~ /(config error: mail loops back to me)/) {
		return $1;
        } elsif ($status =~ /(Authentication-Warning:).*[a-fA-F0-9\.\:]+(.*)/) {
		return "$1 $2";
        } elsif ($status =~ /(Authentication-Warning: [^:]+: [^\s]+ set sender) to .*(using -f)/) {
		return "$1 $2";
	} elsif ($status =~ /Losing .* savemail panic/) {
		return "Losing message, savemail panic";
        } elsif ($status =~ /(probable open proxy)/) {
		return $1;
        } elsif ($status =~ /(Too many hops)/) {
		return $1;
        } elsif ($status =~ /^(.*come back) in/) {
		return "$1 later";
        } elsif ($status =~ /^(setsender: [^:]+):/) {
		my $ret = $1;
		$ret =~ s/ SIZE=.*//;
		return $ret . " attack attempt";
        } elsif ($status =~ /(User unknown)/) {
		return $1;
        } elsif ($status =~ /(Greylisted), see http.*/) {
		return $1;
        } elsif ($status =~ /(improper command pipelining after RCPT).*/) {
		return $1;
        } elsif ($status =~ /(can't identify domain) in.*/) {
		return $1;
        } elsif ($status =~ /.*(Illegal address syntax).*( in .* command)/) {
		return "$1$2";
        } elsif ($status =~ /(improper command pipelining after HELO)/i) {
		return $1;
        } elsif ($status =~ /(You are still greylisted)/i) {
		return $1;
        } elsif ($status =~ /(Domain of sender address) ([^\s]+) (.*)/i) {
		return "$1 $3: $2";
	} elsif ($status =~ /\d{3} \d\.\d\.\d <[^>]+>[:\s\.]*(.*)/) {
		return $1;
	} elsif ($status =~ /\d{3} \d\.\d\.\d [^@]+\@[^\s]+ (.*)/) {
		return $1;
	} elsif ($status =~ /^\d{3} \d\.\d\.\d (.*?) (has exceeded .*)/) {
		return $2;
	} elsif ($status =~ /^\d{3} \d\.\d\.\d ([^\.]+)/) {
		my $str = $1;
		$str =~ s/://;
		return $str;
	} elsif ($status =~ /^\d\.\d\.\d (.*)/) {
		return $1;
	}
	$status =~ s/\.\.\..*//;
	$status =~ s/ with .*//;
	$status =~ s/ by .*//;
	$status =~ s/ \(.*//;
	$status =~ s/ '.'//g;
	$status =~ s/\.$//;
	$status =~ s/;.*$//;

	if ($status =~ /(\d{3} \d\.\d\.\d).*[^a-z\s:\.]+.*/i) {
		return $1;
	}

	return $status;
}

####
# Routine to remove any spercific part of rejected status line to report
# generic status
####
sub clear_rejection_status
{
	my $status = shift;

	if ($status =~ /(\d{3} \d\.\d\.\d)/) {
		return $1;
	}
	$status =~ s/^.*reject=//;
	$status =~ s/^.*\.\.\.\s*//;
	$status =~ s/[^\d]\..*$//;

	return $status;
}


####
# Check wether the current parsed line has alread been parsed.
# return 1 if timestamp of log line is lower (already parsed)
# return 0 if timestamp from log is upper (line not already parsed)
####
sub incremental_check
{
	my ($last_parsed, $old_last_parsed) = @_;

	my $current_year = 0;
	if ($CONFIG{DEFAULT_YEAR}) {
		$current_year = $CONFIG{DEFAULT_YEAR}
	} else {
		$current_year = (localtime(time))[5]+1900;
	}

	# set year date part of the current last parsed line from log file
	my ($month,$day,@f) = split(/[\s\:]+/, $last_parsed, 5);
	$f[0] = sprintf("%02d",$f[0]);
	$f[1] = sprintf("%02d",$f[1]);
	$f[2] = sprintf("%02d",$f[2]);
	my $log_date = $current_year . $MONTH_TO_NUM{"$month"} . sprintf("%02d",$day) . "$f[0]$f[1]$f[2]";

	# set year part of the last date from previous run stored in LAST_PARSED file
	my ($old_month,$old_day,@old_f) = split(/[\s\:]+/, $old_last_parsed, 5);
	$old_f[0] = sprintf("%02d",$old_f[0]);
	$old_f[1] = sprintf("%02d",$old_f[1]);
	$old_f[2] = sprintf("%02d",$old_f[2]);
	my $old_date = $current_year . $MONTH_TO_NUM{"$old_month"} . sprintf("%02d",$old_day) . "$old_f[0]$old_f[1]$old_f[2]";

	# Assume that date in the future are in fact logs from previous year so
	# substract one year to datetime or use the one given at command line.
	if (!$CONFIG{DEFAULT_YEAR}) {
		$current_year -= 1;
	}
	if ($log_date > $CURRENT_TIME) {
		$log_date =~ s/^\d{4}//;
		$log_date = $current_year . $log_date;
	}
	if ($old_date > $CURRENT_TIME) {
		$old_date =~ s/^\d{4}//;
		$old_date = $current_year . $old_date;
	}

	# The current line has already been parsed. You can not parse data that
	# are older than the last run of sendmailanalyzer
	return 1 if ($log_date < $old_date);

	return 0;
}

# Generate a unique identifier
sub get_uniqueid
{

	my $u_id = '';
	while (length($u_id) < 16) {
		my $c = chr(int(rand(127)));
		if ($c =~ /[a-zA-Z0-9]/) {
			$u_id .= $c;
		}
	}

	return 'FaKe' . $u_id;
}

sub clean_globals
{
	%SYSERR = ();
	%DSN = ();
	%FROM = ();
	%TO = ();
	%REJECT = ();
	%SPAM = ();
	%VIRUS = ();
	%ERRMSG = ();
	%OTHER = ();
	%SPAMDETAIL = ();
	%AUTH = ();
	%GREYLIST = ();
	%POSTGREY = ();
	%MSGID = ();
	%SKIPMSG = ();
	%POSTFIX_PLUGIN_TEMP_DELIVERY = ();
	%AMAVIS_ID = ();
	%STARTTLS = ();
	%SPAMPD = ();
	%KEEP_TEMPORARY = ();
}


