#!/usr/bin/perl

use strict;
use Data::Dumper;

# die values geben an, wie oft das pattern auftreten darf, bevor das script einen fehler annimmt
# und den gesamten datenstrom doch durchreicht.
# 
# $Id$

my $filters      = "unset";
my $filterfile   = undef;
my @Addrulefiles = ();
my $help     = 0;
my $isemail  = 0;
my $echo_hdr = 0;
my $echo_bdy = 0;
my $all_err  = 0;
my $set_subj  = 0;
my $set_err   = 0;

doopt(\@ARGV, {
    # use "-x parameter"
    -f => sub { $filters            = shift @{$_[0]} },
    -F => sub { $filterfile         = shift @{$_[0]} },
    -a => sub { push @Addrulefiles,   shift @{$_[0]} },
    # use "-x" as a flag
    -h => sub { $help         = 1              },
    -m => sub { $isemail      = 1              },
    -M => sub { $echo_hdr     = 1              },
    -A => sub { $all_err      = 1              },
	-s => sub { $set_subj     = 1              },
	-e => sub { $set_err      = 1              },
	-b => sub { $echo_bdy     = 1              },
});

my $pat_raw = {

	"sophos" => [
		'^$'   => "u",  # unlimited
		'^SWEEP virus detection utility$' => 1,
		'^Version [0-9.]+ \[Linux/AMD64\]$' => 1,
		'^Virus data version [0-9.]+, [A-Z][a-z]+ \d{4}$' => 1,
		'^Includes detection for \d+ viruses, Trojans and worms$' => 1,
		'^Copyright \(c\) 1989-\d{4} Sophos Group\. All rights reserved\.$' => 1,
		'^System time [0-9:]+, System date \d+ [A-Z][a-z]+ \d{4}$' => 1,
		#'^Command line qualifiers are: -p=/tmp/viruscheck --no-reset-atime -nc -di -remove -archive -all$' => 1,
		#'^Command line qualifiers are: -p=/var/log/viruscheck --no-reset-atime -nc -di -remove -archive -all$' => 1,
		'^Command line qualifiers are:( -\S+)+$' => 1,
		'^IDE directory is: /usr/local/sav$' => 1,
		'^Using IDE file [a-z0-9-]+\.ide$' =>  "u", # unlimited
		'^Quick Sweeping$' => 1,
		'^Could not open ' => "u", # unlimited
		'^\d+ files swept in (\d+ hours, )?\d+ minutes and \d+ seconds\.$' => 1,
		'^\d+ errors were encountered\.$' => 1,
		'^No viruses were discovered\.$' => 1,
		'^End of Sweep\.$' => 1,
	],

	"rsync-rsnapshot" => [
		'^$'   => "u",  # unlimited
		'^send_files failed to open .*: Stale NFS file handle$'     => 5,
		'^send_files failed to open .*: No such file or directory$' => 5,
		'^rsync: mknod ".*" failed: Invalid argument \(22\)$'       => 5,
		'^rsync error: some files could not be transferred \(code 23\) at main.c\(\d+\) \[generator=3\.0\.\d+\]$' => 1,
		'^WARNING: Some files and/or directories in .* only transferred partially during rsync operation$'       => 1,

		'^file has vanished: ".*"$' => 5,
		'^rsync warning: some files vanished before they could be transferred \(code 24\) at main.c\(\d+\) \[(sender|generator)=3\.0\.\d+\]$' => 1,
		'^WARNING: Some files and/or directories in .* vanished during rsync operation$' => 1,
	],

	"rsnapshot-backup" => [
		# same as "rsync-rsnapshot"
		'^$'   => "u",  # unlimited
		'^send_files failed to open .*: Stale NFS file handle$'     => 5,
		'^send_files failed to open .*: No such file or directory$' => 5,
		'^rsync: mknod ".*" failed: Invalid argument \(22\)$'       => 5,
		'^rsync error: some files could not be transferred \(code 23\) at main.c\(\d+\) \[generator=3\.0\.\d+\]$' => 1,
		'^WARNING: Some files and/or directories in .* only transferred partially during rsync operation$'       => 1,

		'^file has vanished: ".*"$' => 5,
		'^rsync warning: some files vanished before they could be transferred \(code 24\) at main.c\(\d+\) \[generator=3\.0\.\d+\]$' => 1,
		'^WARNING: Some files and/or directories in .* vanished during rsync operation$' => 1,
		# /same as "rsync-rsnapshot"

		'^------ BACKUP OF '."'" => "u",
		'^----------------------------' => "u",
	],

	"mysql-zrm" => [
		'^$'   => "u",  # unlimited
		'^schedule:INFO: ZRM for MySQL Community Edition - version ' => 1,
		'^Logging to /var/log/mysql-zrm/mysql-zrm-scheduler.log$'    => 1,
		'^backup:INFO: ZRM for MySQL Community Edition - version '   => 1,
		'^(hour|dai|week|month)lyrun:backup:INFO: '                  => 120,
		'^([a-zA-Z0-9-+_]+ )*mysql ([a-zA-Z0-9-+_]+ )*$'             => 1,
		'^/usr/bin/mysql-zrm started successfully$'                  => 1,
		'^(hour|dai|week|month)lyrun:backup:WARNING: Binary logging is off.$'  => 1,
		'^purge:INFO: ZRM for MySQL Community Edition - version '    => 1,
		'^purge:INFO: Purging Backup /'                              => 5,
		'^Slave stopped$'                                            => 1,
		'^Slave started$'                                            => 1,
		'^}$'                                                        => 3,
		'^src = /var/log/mysql/mysql-bin.\d+$'                       => 3,
		'^k($| = /)'                                                 => 3,
		'^0 0$'                                                      => 1,
		'^logical-parallel=0$'                                       => 1,
		'^Backup set='                                               => 1,
		'^Backup date='                                              => 1,
		'^Backup level=\d+$'                                         => 1,
		'^Logical Databases=([a-zA-Z0-9-+_]+ )*$'                    => 1,
		'^Backup size=\d+\.\d+ [kKMG]B$'                             => 1,
		'^Backup time=\d\d:\d\d:\d\d$'                               => 1,
		'^Backup status=Backup succeeded$'                           => 1,
		'^#mysql50#lost\+found'                                      => 1,
		'^tar: --same-order option cannot be used with -c'           => 1, # tar changed args processing, error affects master/slave backups (only)
		'^ Try \'tar --help\' or \'tar --usage\' for more information.'  => 1, # same as above
		'^Slave.*started'                                            => 1,
	],

	"mysqloptimize" => [
		'^[A-ZUa-z0-9_]+\.[A-ZUa-z0-9_]+\s+(OK|Table is already up to date)$'     => "u",
		'^[A-ZUa-z0-9_]+\.[A-ZUa-z0-9_]+$'                                        => "u",
		q/^note     : The storage engine for the table doesn't support optimize$/ => "u",
		'^note     : Table does not support optimize, doing recreate \+ analyze instead$' => "u",
		'^status   : OK$'                                              => "u",
	],

	"webalizer" => [
		'^$'   => "u",  # unlimited
		'^Webalizer V[0-9-.]+ \([^)]+\) locale: [A-Za-z_@]{1,10}$'     => 1,
		'^Using logfile /\S+ \(clf\)$'                                 => 1,
		'^Using default GeoIP database$'                               => 1,
		'^Creating output in /\S+$'                                    => 1,
		'^Hostname for reports is \'[A-Za-z0-9-.]+\'$'                 => 1,
		'^Reading history file\.\.\. webalizer.hist$'                  => 1,
		'^Reading previous run data\.\. webalizer.current$'            => 1,
		'^\d+ records \(\d+ ignored\) in \d+.\d+ seconds(, \d+/sec)?$' => 1,
	],

	"spamassassin" => [
		'^$'   => "u",  # unlimited
		'^bayes: synced databases from journal in \d+ seconds: \d+ unique entries \(\d+ total entries\)$'   => 1,
	],

	"ntpdate" => [
		'^$'   => "u",  # unlimited
		'^[ 0123]\d [A-Z][a-z]{2} \d\d:\d\d:\d\d ntpdate\[\d{1,5}\]: step time server \S+ offset [01]\.\d+ sec$'   => 1, # catches < 2 sec
	],

	"bacula" => [
		'^$'   => "u",  # unlimited
		'^=$'  => 1,  
	],

	"darbackup" => [
		'^$'   => "u",  # unlimited
		'^ --------------------------------------------$'  => 4,  
		'^ \d+ inode\(s\) saved$'  => 1,  
		'^ with \d+ hard link\(s\) recorded$'  => 1,  
		'^ with \d+ hard link\(s\) recorded$'  => 1,  
		'^ \d+ inode\(s\) changed at the moment of the backup$' => 1,
		'^ \d+ inode\(s\) not saved \(no inode/file change\)$' => 1,
		'^ \d+ inode\(s\) failed to save \(filesystem error\)$' => 1,
		'^ \d+ inode\(s\) ignored \(excluded by filters\)$' => 1,
		'^ \d+ inode\(s\) recorded as deleted from reference backup$' => 1,
		'^ Total number of inodes? considered: \d+$' => 1,
		'^ EA saved for \d+ inode\(s\)$' => 1,
		'^dar-exit-status: 0$' => 1,
		'^dar-exit-status: 11$' => 1,

		'^WARNING! File modified while reading it for backup: .*\.log$' => 5,
		'^WARNING! File modified while reading it for backup: .*\.rrd$' => 5,
		'^WARNING! File modified while reading it for backup: .*\.jnl$' => 5,
		'^WARNING! File modified while reading it for backup: .*\.LOG$' => 5,
		'^WARNING! File modified while reading it for backup: /var/log/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/cache/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /tmp/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/lib/vservers/[^/]+/var/cache/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/lib/vservers/[^/]+/var/log/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/lib/vservers/[^/]+/var/lib/amavis/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/lib/vservers/[^/]+/var/lib/mysql/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/lib/vservers/[^/]+/var/lib/postgresql/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/lib/vservers/[^/]+/var/lib/cyrus/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/lib/vservers/[^/]+/var/lib/nagios3/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/lib/vservers/[^/]+/var/lib/nagios-pnp/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/lib/vservers/[^/]+/var/mail/.*$' => 5,
		'^WARNING! File modified while reading it for backup: /var/lib/vservers/[^/]+/tmp/.*$' => 5,
		'^File has been removed while reading it for backup: /' => 5,
	],

	# kerberos replication
	"kprop" => [
		'^Database propagation to.*SUCCEEDED$' => "u",
		'^[1-9]\d* bytes sent\.$' => "u",
	],

	# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=818349
	"exim201603" => [
		'^/etc/cron.daily/exim4-base:$' => 1,
		'^LOG: MAIN$' => 1,
		'^  Warning: purging the environment.$' => 1,
		'^ Suggested action: use keep_environment.$' => 1,
	],
};

# set up pattern hash for a c)ounter, an a)llowance value and a s)ource indicator
my $pat = {};
for my $i ( keys %$pat_raw ) {
	for(my $j=0; $j <= $#{$pat_raw->{$i}}; $j+=2) {
		# $pat->{$i}->{ $pat_raw->{$i}->[$j] } = $pat_raw->{$i}->[$j+1];
		$pat->{$i}->{ $pat_raw->{$i}->[$j] } = { 
			a => $pat_raw->{$i}->[$j+1],
			c => $pat_raw->{$i}->[$j+1],
			s => "int:$i:".($j+2)/2,
		};
	}
}

$help && usage(keys %$pat) && exit;

if ( $filterfile ) {
	my $rules = readrulesfile ($filterfile);
	$pat->{$filters} = $rules;
} 

if ( @Addrulefiles ) {
	for my $i ( @Addrulefiles ) {
		my $rules = readrulesfile ($i);
		$pat->{$filters} = { %{$pat->{$filters}}, %$rules };
	}
}

my @Lines = ();
my @Header = ();

if ( $isemail ) {
	while (my $l = <>) {
		push @Header,$l;
		if ( $l eq "\n" ) {
			last;
		}
	}
}

#dev# print Dumper($pat);
#dev# print "F: $filters\n";
#dev# print join "\n  ","",keys %{$pat->{$filters}},"\n";

my $p_cnt; # $pattern count for reporting
my $p_rul; # $pattern rule spec for reporting
my @ReportM = (
	"Diese Nachricht wurde von $0 nachbearbeitet. Den unbekannten Zeilen wurde\n",
	"diese Zeile vorangestellt: '#### maybe this indicates an error ...'\n",
);
my @ReportS = (
	"Diese Nachricht wurde von $0 nachbearbeitet. Der ersten unbekannten Zeile wurde\n",
	"diese Zeile vorangestellt: '#### maybe this indicates an error ...'\n",
);
my $err_cnt; # error count for reporting
LINE: while (my $l = <>)  {
	push @Lines,$l;
	chomp $l;
	for my $p ( keys %{$pat->{$filters}} ) {
		if ( $l =~ $p ) {
			#dev# print STDERR "match\n";
			#dev# print STDERR "$l\n";
			#dev# print STDERR "$p\n";
			$pat->{$filters}->{$p}->{a} eq "u" and do {
				$pat->{$filters}->{$p}->{c} ++;
				next LINE; #--^
			}; # else:
			$pat->{$filters}->{$p}->{c} --;
			#dev# print "line:  $l\n";
			#dev# print "match: $p (",$pat->{$filters}->{$p}->{c},")\n";

			if ( $pat->{$filters}->{$p}->{c} >= 0 ) {
				#dev# print STDERR ">= 0\n";
				next LINE; #--^
			}
			# for reporting:
			$p_cnt = $pat->{$filters}->{$p}->{a};
			$p_rul = $pat->{$filters}->{$p}->{s};
		}
	}
	$err_cnt ++;
	my $report = "#### maybe this indicates an error".
		((defined $p_rul)?(" (Ruleset: $p_rul)"):()).
		((defined $p_cnt)?(" (match count > ".$p_cnt.")"):()).
		":\n";
	my $lp = pop @Lines;
	push @Lines, $report;
	push @Lines, $lp;

	if ( $all_err ) {
		next LINE;
	} else {
	    last LINE;
	}
	
} # LINE:

if ( $echo_hdr ) {
	if ( $err_cnt > 0 and $set_subj ) {
		for my $i ( @Header ) {
			$i =~ s/^Subject: /Subject: [cserr] /;
		}
	}
	print @Header;
}
if ( $err_cnt > 0 ) {
	if ( $all_err ) {
		print 
			@ReportM,
			"\n",
			@Lines,
	} else {
		print 
			@ReportS,
			"\n",
			@Lines,
			<>;
	}
	if ( $set_err ) {
		exit 1;
	}
} else {
	if ( $echo_bdy ) {
		print @Lines;
	}
}

###########################
## subroutine collection
###########################
#
# process command line arguments
# usage:
#
# doopt(\@ARGV, {
#     # use "-x parameter"
#     -o => sub { $outFileBase  = shift @{$_[0]} },
#     # use "-x" as a flag
#     -f => sub { $flag         = 1              },
# });
#
#
#

sub doopt {
    my ($argvRef, $optRef ) = @_;
    my @ArgBak = ();
    while ( my $arg = shift @$argvRef ) {
        if ( defined $optRef->{$arg} ) {
            $optRef->{$arg}->($argvRef);
        } else {
            push @ArgBak, $arg;
        }
    }
    @$argvRef = @ArgBak;
}

sub usage {
	my @P = @_;

	print "usage: $0 [ -e ] [ -s ] [-m [ -M ] [ -b ]] [ -A ] { -f { ",join(" | ",@P)," } | -F file } [ -a file [-a file ...]] [ file ]\n";
	print "\n";
    print "Some stupid cron jobs throw around useless messages.\n";
    print "Just pipe them through cron-silencer to shut them up.\n";
    print "C. filters out well-known lines and only if there are\n";
    print "unknown ones (presumably errors) you get the full message,\n";
    print "with the incriminated line marked.\n";
    print "-f  filter through intrinsic filter rules\n";
    print "-F  filter through filter rules from file\n";
    print "-a  filter also through rules from file (may be specified multiple times)\n";
    print "-A  report all errors (default: only the first error is reported)\n";
    print "-m  input is an e-mail, skip until (and including) first empty line\n";
    print "-M  echo email headers to output\n";
    print "-b  echo email body to output\n";
    print "-e  set errorcode to 1 if any errors were found (default: 0)\n";
    print "-s  prefix email-subject with '[cserr]' if any errors were found (default: no prefix)\n";
	print "\n";
	print "example:   0 22 * * * root mysql-zrm 2>&1 | cron-silencer -f mysql-zrm\n";
	print "example:   0 22 * * * root mysql-zrm 2>&1 | cron-silencer -f mysql-zrm -a /etc/cron-silencer.d/enhance-zrm.cnf\n";
	print "example:   0 22 * * * root ownscript 2>&1 | cron-silencer -F /etc/cron-silencer/ownscript.cnf\n";
	print "\n";
}

sub readrulesfile {
	my $rulefilename = $_[0];
	my $re = {};
	open F,$rulefilename or die "can't open '$rulefilename': $!";
	while ( my $l = <F>) {
	    next if $l =~ /^(#|$)/;
		my ($r,$c) = $l =~ m#^/(.*)/\s*=>\s*(u|\d+)\s*$#;
        $re->{$r} = { a => $c, c => $c, s => "ext:$rulefilename:$." };
	}
	close F;

	return $re;
}
