5 Yaala::Parser::Sendmail
9 Parser for sendmail log.
13 Each message in sendmail log forms several records (lines):
14 one record with 'from=' field, and one or several with 'to=' field.
16 Parser joins each 'from'-part with 'to'-part by message log id.
17 However, if there're several recipients, result will be several records for the same message:
19 When message is first time countd, datafield 'uniq' is set to 1.
20 This is usefull to calculate total count/traffic or traffic by type.
21 But if you count total by recipients, using this key (as WHERE uniq=='1')
22 will make yaala ignore all recipients of a message, but the first.
24 Grand total (extra) is counted unique and only with stat=/Sent.*/
28 sendmail_aliases - aliases file used to resolve adresses
29 sendmail_localdomain - local domain to remove from adresses
30 sendmail_localrelay - IP regexp to determine incoming/outgoing/local traffic, egg '192.168.1.\d+'
80 =head2 Aggregation-Fields
92 =head2 Additional Notes
94 timedate is splited to I<date> and I<hour>, as usual, year is taken from
95 current date. I<rrelay> is relay field from to-part
97 I<uniq> is set to 1 when message first time counted.
99 I<type> = "I","O","L","R" for incoming, outgoing, local and relay traffic. It
100 is determined using fields 'mailer' and 'relay'. (Thus, only applied to
101 sent/recieved messages)
107 =item Properly resolve multiple aliases.
109 =item Split non-local multiple recipients
115 qMax E<lt>qmax-at-mediasoft.ruE<gt>
121 use vars qw(%DATAFIELDS);
124 use Yaala::Parser::WebserverTools qw(%MONTH_NUMBERS);
125 use Yaala::Data::Persistent qw#init#;
126 use Yaala::Config qw#get_config#;
128 @Yaala::Parser::EXPORT_OK = qw(parse extra %DATAFIELDS);
129 @Yaala::Parser::ISA = ('Exporter');
131 our $EXTRA = init ('$EXTRA', 'hash');
135 if (!defined ($EXTRA->{'totalcount'})) { $EXTRA->{'totalcount'} = {I=>0, O=>0, L=>0}; }
136 if (!defined ($EXTRA->{'totalamount'})) { $EXTRA->{'totalamount'} = {I=>0, O=>0, L=>0}; }
137 if (!defined ($EXTRA->{'start'} )) { $EXTRA->{'start'} = undef; }
138 if (!defined ($EXTRA->{'end'} )) { $EXTRA->{'end'} = undef; }
170 # This needs to be done at runtime, since Data uses Setup which relies on
171 # %DATAFIELDS to be defined -octo
172 require Yaala::Data::Core;
173 import Yaala::Data::Core qw#store#;
175 my $VERSION = 'v 1.1$';
176 print STDERR $/, __FILE__, ": $VERSION" if ($::DEBUG);
179 my $aliasfile = get_config("sendmail_aliases");
181 print STDERR $/, __FILE__, ": Loaded aliases from $aliasfile" if ($::DEBUG);
182 load_aliases($aliasfile);
185 our $localdomain = get_config("sendmail_localdomain");
186 our $localrelay = get_config("sendmail_localrelay");
187 print STDERR $/, __FILE__, ": Local relay: $localrelay" if ($::DEBUG);
188 $localrelay = qr/\[$localrelay\]/;
190 our %RECF = (); # all pending from-parts
191 our %RECT = (); # all pending to-parts
197 my $line = shift or return undef;
198 if ( $line =~ s/^(...\s*\d+ \d\d:\d\d:\d\d) [\w-]+ sm-mta.*\[\d+\]: ([a-zA-Z0-9]{14}): // )
202 if( $line =~ /^from=/ ) {
203 my $rec = parseline($line);
204 $rec->{'from'} = resolve_alias($rec->{'from'});
205 $RECF{$id} = { datetime=>$datetime, %$rec };
208 elsif( $line =~ /^to=/ ) {
209 my $rec = parseline($line);
210 $rec->{'to'} = resolve_alias($rec->{'to'});
211 $RECT{$id} = { datetime=>$datetime, %$rec };
215 # some heaers mangling or mail filters log lines
224 foreach (split(/,\s+/,$line)) {
225 if( m/(.*?)=(.*)/ ) {
226 $rec{$1}=$2 if exists $DATAFIELDS{$1};
235 return unless ( $RECF{$id} and $RECT{$id} );
236 # rename relay in TO-part into rrelay
237 $RECT{$id}->{'rrelay'} = $RECT{$id}->{'relay'} if $RECT{$id}->{'relay'};
238 delete $RECT{$id}->{'relay'};
240 my %rec = ( %{$RECF{$id}}, %{$RECT{$id}} );
242 #print STDERR "\nRECT K: ",join("; ", keys %{$RECT{$id}});
243 #print STDERR "\nRECT V: ",join("; ", values %{$RECT{$id}});
244 #print STDERR "\nRECF K: ",join("; ", keys %{$RECF{$id}});
245 #print STDERR "\nRECF V: ",join("; ", values %{$RECF{$id}});
247 $rec{'datetime'} =~ /(\w\w\w)\s*(\d+) (\d\d):\d\d:\d\d/;
248 my ($month,$day,$hour) = ($1,$2,$3);
249 $month = $MONTH_NUMBERS{$month};
250 my $year = [localtime(time)]->[5]+1900; # current year
251 my $date = sprintf("%04u-%02u-%02u", $year, $month, $day);
254 $combined{'date'}=$date;
255 $combined{'hour'}=$hour;
256 $combined{'count'}=1;
257 $combined{'uniq'} = (exists($COUNTED{$id}) ? 0 : 1);
258 $combined{'to'} =~ s/\</\</g;
259 $combined{'to'} =~ s/\>/\>/g;
260 $combined{'from'} =~ s/\</\</g;
261 $combined{'from'} =~ s/\>/\>/g;
262 $combined{'stat'} =~ s/^((\w+)(\s+\w+)*).*$/$1/;
265 if( $localrelay and $combined{'relay'} and $combined{'mailer'}) {
271 ( $combined{'relay'} =~ $localrelay ) ?
272 ( ( $combined{'mailer'} eq 'local' ) ? 'L' : 'O' ) :
273 ( ( $combined{'mailer'} eq 'local' ) ? 'I' : 'R' ) ;
277 $combined{'type'} = $type;
279 unless( $COUNTED{$id} ) {
281 $EXTRA->{'totalcount'}->{$type}++;
282 $EXTRA->{'totalamount'}->{$type}+=$combined{'size'};
285 if( not defined $EXTRA->{'start'} or $month < $EXTRA->{'start'}->{'m'} or $day < $EXTRA->{'start'}->{'d'} ) {
286 $EXTRA->{'start'}->{'m'} = $month;
287 $EXTRA->{'start'}->{'d'} = $day;
289 if( not defined $EXTRA->{'end'} or $month > $EXTRA->{'end'}->{'m'} or $day > $EXTRA->{'end'}->{'d'} ) {
290 $EXTRA->{'end'}->{'m'} = $month;
291 $EXTRA->{'end'}->{'d'} = $day;
294 #print STDERR "\nParsed $id: ",join(";",map("$_=".$combined{$_}, sort keys %combined));
305 if( open(F,"<$file") ) {
308 next unless( /^([^ ]*)\s*:\s*([^, ]*)\s*$/ );
309 $ALIASES{lc($1)}=lc($2);
318 my $addr = lc($alias);
320 $addr =~ s/\@$localdomain// if( $localdomain );
321 if( $ALIASES{$addr} ) {
322 $addr = $ALIASES{$addr};
329 $::EXTRA->{'0. Begin/End date'} = sprintf "%02d/%02d - %02d/%02d",
330 $EXTRA->{'start'}->{'d'},
331 $EXTRA->{'start'}->{'m'},
332 $EXTRA->{'end'}->{'d'},
333 $EXTRA->{'end'}->{'m'};
335 if ($EXTRA->{'totalcount'}->{'I'})
337 $::EXTRA->{'1. Incoming mail'} = sprintf ("%5d / %12d Bytes",
338 $EXTRA->{'totalcount'}->{'I'}, $EXTRA->{'totalamount'}->{'I'});
340 if ($EXTRA->{'totalcount'}->{'O'})
342 $::EXTRA->{'2. Outgoing mail'} = sprintf ("%5d / %12d Bytes",
343 $EXTRA->{'totalcount'}->{'O'}, $EXTRA->{'totalamount'}->{'O'});
345 if ($EXTRA->{'totalcount'}->{'I'})
347 $::EXTRA->{'3. Local mail'} = sprintf ("%5d / %12d Bytes",
348 $EXTRA->{'totalcount'}->{'L'}, $EXTRA->{'totalamount'}->{'L'});
350 if ($EXTRA->{'totalcount'}->{'R'})
352 $::EXTRA->{'4. Relayed mail'} = sprintf ("%5d / %12d Bytes",
353 $EXTRA->{'totalcount'}->{'R'}, $EXTRA->{'totalamount'}->{'R'});
355 if ($EXTRA->{'totalcount'}->{'UNDEF'})
357 $::EXTRA->{'5. Unknown mail'} = sprintf ("%5d / %12d Bytes",
358 $EXTRA->{'totalcount'}->{'UNDEF'}, $EXTRA->{'totalamount'}->{'UNDEF'});