package Yaala::Parser;

# ncsa.pm was patched to support wn-server by M. Feenstra on 20/09/2001

use strict;
use warnings;
use vars qw(%DATAFIELDS);

use Exporter;
use Yaala::Parser::WebserverTools qw#%MONTH_NUMBERS detect_referer detect_browser
	detect_os extract_data#;
use Yaala::Data::Persistent qw#init#;

@Yaala::Parser::EXPORT_OK = qw(parse extra %DATAFIELDS);
@Yaala::Parser::ISA = ('Exporter');

our $LASTDATE = init ('$LASTDATE', 'scalar');
our $EXTRA = init ('$EXTRA', 'hash');

if (!$$LASTDATE) { $$LASTDATE = 0; }
if (!defined ($EXTRA->{'total'})) { $EXTRA->{'total'} = 0; }
if (!defined ($EXTRA->{'days'} )) { $EXTRA->{'days'}  = {}; }
if (!defined ($EXTRA->{'search_terms'} )) { $EXTRA->{'search_terms'} = {}; }

%DATAFIELDS = (	
	host		=> 'key:host',
	user		=> 'key',
	date		=> 'key:date',
	hour		=> 'key:hour',
	tld		=> 'key',
	file		=> 'key',
	status		=> 'key:numeric',
	browser		=> 'key',
	os		=> 'key',
	referer		=> 'key:url',
	virtualhost	=> 'key',
	
	bytes		=> 'agg:bytes',
	requests	=> 'agg'
);

# This needs to be done at runtime, since Data uses Setup which relies on
# %DATAFIELDS to be defined  -octo
require Yaala::Data::Core;
import Yaala::Data::Core qw#store#;

my $VERSION = '$Id: Wnserver.pm,v 1.9 2003/12/07 16:48:59 octo Exp $';
print STDERR $/, __FILE__, ": $VERSION" if ($::DEBUG);

return (1);

sub parse
{
	my $line = shift or return undef;
	
	if ($line =~ /^(\S+) (\S+) (\S+) \[([^\]]+)\] "([^"]+)" (\d+) (\d+) <[^>]*><([^>]*)> <([^>]*)> <([^>]*)> <([^>]*)>$/)
	{
# Initialize the variables that we can get out of
# each line first..
		my ($host, $ident, $user, $date, $request, $status,
			$bytes, $browser, $referer, $cookie, $virtual) =
		($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);

# And now initialize all the variables we will use
# to get more information out of each field..
		my ($day, $month, $year, $hour, $minute, $second) =
			$date =~ m#(\d\d)/(\w{3})/(\d{4}):(\d\d):(\d\d):(\d\d)#;

		$month = $MONTH_NUMBERS{$month};
		$date = sprintf("%04u-%02u-%02u", $year, $month, $day);

		{
			my $tmp = int (sprintf ("%04u%02u%02u%02u%02u%02u",
					$year, $month, $day, $hour, $minute, $second));
			
			if ($tmp < $$LASTDATE)
			{
				print STDERR $/, __FILE__, ": Skipping.. ($tmp < $$LASTDATE)" if ($::DEBUG & 0x0200);
				next;
			}
			else { $$LASTDATE = $tmp; }
		}
		
		my ($method, $file, $params);
		if ($request =~ m#(\S+) ([^ \?]+)\??(\S*)#)
		{
			$method = $1;
			$file = $2;
			$params = (defined ($3) ? $3 : '');
		}
		else
		{
			print STDERR $/, __FILE__, ": Malformed request: ``$request''." if ($::DEBUG);
			return (0);
		}
		
		if (($user ne '-') and ($status >= 400) and ($status < 500))
		{
			$user = '*INVALID*';
		}

		if ($user eq '-') { $user = '*UNKNOWN*'; }
		if ($bytes eq '-') { $bytes = 0; }

		my $tld;
		if ($host =~ m/\.([a-z]{2,})$/i)				
		{
			$tld = lc ($1);
		}
		else
		{       
			$tld = '*UNRESOLVED*';
		}

		my $os = detect_os ($browser);
		my $browser_name = detect_browser ($browser);
		my @search_terms = extract_data ($referer);
		if ($referer eq '-') { $referer = ''; }

		$EXTRA->{'total'}++;
		$EXTRA->{'days'}{$date}++;

		if (scalar @search_terms)
		{
			print $/, __FILE__, ": Search Terms: ",
				join (' ', @search_terms)
				if ($::DEBUG & 0x1000);
			
			$EXTRA->{'search_terms'}{$_}++ for (@search_terms);
		}

		my %combined = (
					'host'		=>	$host,
					'user'		=>	$user,
					'date'		=>	$date,
					'hour'		=>	$hour,
					'browser'	=>	$browser_name,
					'os'		=>	$os,
					'tld'		=>	$tld,
					'file'		=>	$file,
					'referer'	=>	$referer,
					'status'	=>	$status,
					'bytes'		=>	$bytes,
					'virtualhost'	=>	$virtual,
					'requests'	=>	1
				);
		store (\%combined);
	}
	elsif ($::DEBUG)
	{
		chomp ($line);
		print STDERR $/, __FILE__, ": Unable to parse: ``$line''";
	}
}

sub extra
{
	my ($average, $days) = (0, 0);
	
	$days = scalar (keys (%{$EXTRA->{'days'}}));
	return (0) unless ($days);
	
	$average = sprintf ("%.1f", ($EXTRA->{'total'} / $days));

	$::EXTRA->{'Total requests'} = $EXTRA->{'total'};
	$::EXTRA->{'Average requests per day'} = $average;
	$::EXTRA->{'Reporting period'} = "$days days";
	
	my @sorted_terms = sort
		{ $EXTRA->{'search_terms'}{$b} <=> $EXTRA->{'search_terms'}{$a} }
		(keys %{$EXTRA->{'search_terms'}});
	
	if (@sorted_terms)
	{
		my $max = $EXTRA->{'search_terms'}{$sorted_terms[0]};
		my @scalar_terms = ();
		
		while (@sorted_terms and
			($EXTRA->{'search_terms'}{$sorted_terms[0]} / $max) > 0.1)
		{
			$_ = shift (@sorted_terms);
			
			push (@scalar_terms,
				sprintf ("%s (%u)",
					$_, $EXTRA->{'search_terms'}{$_})
			);
		}
		$::EXTRA->{'Search terms used'} = join ("<br />\n      ", @scalar_terms);

		if (@sorted_terms)
		{
			my $skipped = scalar (@sorted_terms);
			$::EXTRA->{'Search terms used'} .= "<br />\n      $skipped more skipped";
		}
	}
}
