package WebPAC::Validate;

use warnings;
use strict;

use blib;

use base 'WebPAC::Common';
use File::Slurp;
use List::Util qw/first/;
use Data::Dump qw/dump/;
use WebPAC::Normalize qw/_pack_subfields_hash/;
use Storable qw/dclone/;

=head1 NAME

WebPAC::Validate - provide simple validation for records

=head1 VERSION

Version 0.12

=cut

our $VERSION = '0.12';

=head1 SYNOPSIS

This module provide a simple way to validate your file against a simple
configuration file in following format:

  # field 10 doesn't have any subfields
  10
  # same with 101
  101
  # field 200 have valid subfields a-g
  # and field e is repeatable
  200 a b c d e* f g
  # field 205 can have only subfield a
  # and must exists
  205! a
  # while 210 can have a c or d
  210 a c d
  # field which is ignored in validation
  999-

=head1 FUNCTIONS

=head2 new

Create new validation object

  my $validate = new WebPAC::Validate(
  	path => 'conf/validate/file',
	delimiters => [ ' : ', ' / ', ' ; ', ' , ' ],
	delimiters_path => 'conf/validate/delimiters/file',
  );

Optional parametar C<delimiters> will turn on validating of delimiters. Be
careful here, those delimiters are just stuck into regex, so they can
contain L<perlre> regexpes.

C<path> and C<delimiters_path> can be specified by L<read_validate_file> and
L<read_validate_delimiters> calls.

=cut

sub new {
	my $class = shift;
	my $self = {@_};
	bless($self, $class);

	my $log = $self->_get_logger();

	$self->read_validate_file( $self->{path} ) if ( $self->{path} );

	if ( $self->{delimiters} ) {
		$self->{delimiters_regex} = '(\^[a-z0-9]|' . join('|', @{ $self->{delimiters} }) . ')';
		$log->info("validation check delimiters with regex $self->{delimiters_regex}");
	}

	$self->read_validate_delimiters_file( $self->{delimiters_path} ) if ( $self->{delimiters_path} );

	return $self;
}


=head2 read_validate_file

Specify validate rules file

  $validate->read_validate_file( 'conf/validate/file' );

Returns number of lines in file

=cut

sub read_validate_file {
	my $self = shift;

	my $path = shift || die "no path?";

	my $log = $self->_get_logger();

	my $v_file = read_file( $path ) ||
		$log->logdie("can't open validate path $path: $!");

	my $v;
	delete( $self->{must_exist} );
	delete( $self->{must_exist_sf} );
	delete( $self->{dont_validate} );
	my $curr_line = 1;

	foreach my $l (split(/[\n\r]+/, $v_file)) {
		$curr_line++;

		# skip comments and whitespaces
		next if ($l =~ /^#/ || $l =~ /^\s*$/);

		$l =~ s/^\s+//;
		$l =~ s/\s+$//;

		my @d = split(/\s+/, $l);

		my $fld = shift @d;

		if ($fld =~ s/!$//) {
			$self->{must_exist}->{$fld}++;
		} elsif ($fld =~ s/-$//) {
			$self->{dont_validate}->{$fld}++;
		}

		$log->logdie("need field name in line $curr_line: $l") unless (defined($fld));

		if (@d) {
			$v->{$fld} = [ map {
				my $sf = $_;
				if ( $sf =~ s/!(\*)?$/$1/ ) {
					$self->{must_exist_sf}->{ $fld }->{ $sf }++;
				};
				$sf;
			} @d ];
		} else {
			$v->{$fld} = 1;
		}

	}

	$log->debug("current validation rules: ", dump($v));

	$self->{rules} = $v;

	$log->info("validation uses rules from $path");

	return $curr_line;
}

=head2 read_validate_delimiters_file

  $validate->read_validate_delimiters_file( 'conf/validate/delimiters/file' );

=cut

sub read_validate_delimiters_file {
	my $self = shift;

	my $path = shift || die "no path?";

	my $log = $self->_get_logger();

	delete( $self->{_validate_delimiters_templates} );
	delete( $self->{_delimiters_templates} );

	if ( -e $path ) {
		$log->info("using delimiter validation rules from $path");
		open(my $d, $path) || $log->fatal("can't open $path: $!");
		while(<$d>) {
			chomp($d);
			if (/^\s*(#*)\s*(\d+)\t+(\d+)\t+(.*)$/) {
				my ($comment,$field,$count,$template) = ($1,$2,$3,$4);
				$self->{_validate_delimiters_templates}->{$field}->{$template} = $count unless ($comment);
			} else {
				warn "## ignored $d\n";
			}
		}
		close($d);
		#warn "_validate_delimiters_templates = ",dump( $self->{_validate_delimiters_templates} );
	} else {
		$log->warn("delimiters path $path doesn't exist, it will be created after this run");
	}
	$self->{delimiters_path} = $path;
}

=head2 validate_rec

Validate record and return errors

  my @errors = $validate->validate_rec( $rec, $rec_dump );

=cut

sub validate_rec {
	my $self = shift;

	my $log = $self->_get_logger();

	my $rec = shift || $log->logdie("validate_rec need record");
	my $rec_dump = shift;

	$log->logdie("rec isn't HASH") unless (ref($rec) eq 'HASH');
#	$log->logdie("can't find validation rules") unless (my $r = $self->{rules});
	my $r = $self->{rules};

	my $errors;

	$log->debug("rec = ", sub { dump($rec) }, "keys = ", keys %{ $rec });

	my $fields;

	foreach my $f (keys %{ $rec }) {

		next if (!defined($f) || $f eq '' || $f eq '000');

		# first check delimiters
		if ( my $regex = $self->{delimiters_regex} ) {

			foreach my $v (@{ $rec->{$f} }) {
				my $l = _pack_subfields_hash( $v, 1 );
				my $subfield_dump = $l;
				my $template = '';
				$l =~ s/$regex/$template.=$1/eg;
				#warn "## template: $template\n";

				if ( $template ) {
					$self->{_delimiters_templates}->{$f}->{$template}++;

					if ( my $v = $self->{_validate_delimiters_templates} ) {
						if ( ! defined( $v->{$f}->{$template} ) ) {
							$errors->{$f}->{potentially_invalid_combination} = $template;
							$errors->{$f}->{dump} = $subfield_dump;
						#} else {
						#	warn "## $f $template ok\n";
						}
					}
				}
			}
		}

		next unless ( $r );	# skip validation of no rules are specified

		next if (defined( $self->{dont_validate}->{$f} ));

		# track field usage
		$fields->{$f}++;

		if ( ! defined($r->{$f}) ) {
			$errors->{ $f }->{unexpected} = "this field is not expected";
			next;
		}


		if (ref($rec->{$f}) ne 'ARRAY') {
			$errors->{ $f }->{not_repeatable} = "probably bug in parsing input data";
			next;
		}

		foreach my $v (@{ $rec->{$f} }) {
			# can we have subfields?
			if (ref($r->{$f}) eq 'ARRAY') {
				# are values hashes? (has subfields)
				if (! defined($v)) {
#					$errors->{$f}->{empty} = undef;
#					$errors->{dump} = $rec_dump if ($rec_dump);
				} elsif (ref($v) ne 'HASH') {
					$errors->{$f}->{missing_subfield} = join(",", @{ $r->{$f} }) . " required";
					$errors->{$f}->{dump} = $v;
					next;
				} else {

					my $h = dclone( $v );

					my $sf_repeatable;

					delete($v->{subfields}) if (defined($v->{subfields}));

					my $subfields;

					foreach my $sf (keys %{ $v }) {

						$subfields->{ $sf }++;

						# is non-repeatable but with multiple values?
						if ( ! first { $_ eq $sf.'*' } @{$r->{$f}} ) {
							if ( ref($v->{$sf}) eq 'ARRAY' ) {
								$sf_repeatable->{$sf}++;
							};
							if (! first { $_ eq $sf } @{ $r->{$f} }) {
								$errors->{ $f }->{subfield}->{extra}->{$sf}++;
							}
						}

					}
					if (my @r_sf = sort keys( %$sf_repeatable )) {

						foreach my $sf (@r_sf) {
							$errors->{$f}->{subfield}->{extra_repeatable}->{$sf}++;
							$errors->{$f}->{dump} = _pack_subfields_hash( $h, 1 );
						}

					}

					if ( defined( $self->{must_exist_sf}->{$f} ) ) {
						foreach my $sf (sort keys %{ $self->{must_exist_sf}->{$f} }) {
#warn "====> $f $sf must exist\n";
							$errors->{$f}->{subfield}->{missing}->{$sf}++
								unless defined( $subfields->{$sf} );
						}
					}

				}
			} elsif (ref($v) eq 'HASH') {
				$errors->{$f}->{unexpected_subfields}++;
				$errors->{$f}->{dump} = _pack_subfields_hash( $v, 1 );
			}
		}
	}

	$log->debug("_delimiters_templates = ", sub { dump( $self->{_delimiters_templates} ) } );

	foreach my $must (sort keys %{ $self->{must_exist} }) {
		next if ($fields->{$must});
		$errors->{$must}->{missing}++;
		$errors->{dump} = $rec_dump if ($rec_dump);
	}

	if ($errors) {
		$log->debug("errors: ", $self->report_error( $errors ) );

		my $mfn = $rec->{'000'}->[0] || $log->logconfess("record ", sub { dump( $rec ) }, " doesn't have MFN");
		$self->{errors}->{$mfn} = $errors;
	}

	#$log->logcluck("return from this function is ARRAY") unless wantarray;

	return $errors;
}

=head2 reset

Clean all accumulated errors for this input and remember delimiter templates
for L<save_delimiters_templates>

  $validate->reset;

This function B<must> be called after each input to provide accurate statistics.

=cut

sub reset {
	my $self = shift;

	my $log = $self->_get_logger;

	delete ($self->{errors});

	if ( ! $self->{_delimiters_templates} ) {
		$log->debug("called without _delimiters_templates?");
		return;
	}

	foreach my $f ( keys %{ $self->{_delimiters_templates} } ) {
		foreach my $t ( keys %{ $self->{_delimiters_templates}->{$f} } ) {
			$self->{_accumulated_delimiters_templates}->{$f}->{$t} +=
				$self->{_delimiters_templates}->{$f}->{$t};
		}
	}
	$log->debug("_accumulated_delimiters_templates = ", sub { dump( $self->{_accumulated_delimiters_templates} ) } );
	delete ($self->{_delimiters_templates});
}

=head2 all_errors

Return hash with all errors

  print dump( $validate->all_errors );

=cut

sub all_errors {
	my $self = shift;
	return $self->{errors};
}

=head2 report_error

Produce nice humanly readable report of single error

  print $validate->report_error( $error_hash );

=cut

sub report_error {
	my $self = shift;

	my $h = shift || die "no hash?";

	sub _unroll {
		my ($self, $tree, $accumulated) = @_;

		my $log = $self->_get_logger();

		$log->debug("# ",
			( $tree			? "tree: $tree "					: '' ),
			( $accumulated	? "accumulated: $accumulated "		: '' ),
		);

		my $results;

		if (ref($tree) ne 'HASH') {
			return ("$accumulated\t($tree)", undef);
		}

		my $dump;

		foreach my $k (sort keys %{ $tree }) {

			if ($k eq 'dump') {
				$dump = $tree->{dump};
				#warn "## dump ",dump($dump),"\n";
				next;
			}

			$log->debug("current: $k");

			my ($new_results, $new_dump) = $self->_unroll($tree->{$k},
				$accumulated ? "$accumulated\t$k" : $k
			);

			$log->debug( "new_results: ", sub { dump($new_results) } ) if ( $new_results );

			push @$results, $new_results if ($new_results);
			$dump = $new_dump if ($new_dump);

		}

		$log->debug( "results: ", sub { dump($results) } ) if ( $results );

		if ($#$results == 0) {
			return ($results->[0], $dump);
		} else {
			return ($results, $dump);
		}
	}


	sub _reformat {
		my $l = shift;
		$l =~ s/\t/ /g;
		$l =~ s/_/ /g;
		return $l;
	}

	my $out = '';

	for my $f (sort keys %{ $h }) {
		$out .= "$f: ";
		
		my ($r, $d) = $self->_unroll( $h->{$f} );
		my $e;
		if (ref($r) eq 'ARRAY') {
			$e .= join(", ", map { _reformat( $_ ) } @$r);
		} else {
			$e .= _reformat( $r );
		}
		$e .= "\n\t$d" if ($d);

		$out .= $e . "\n";
	}
	return $out;
}


=head2 report

Produce nice humanly readable report of errors

  print $validate->report;

=cut

sub report {
	my $self = shift;
	my $e = $self->{errors} || return;

	my $out;
	foreach my $mfn (sort { $a <=> $b } keys %$e) {
		$out .= "MFN $mfn\n" . $self->report_error( $e->{$mfn} ) . "\n";
	}

	return $out;

}

=head2 delimiters_templates

Generate report of delimiter tamplates 

  my $report = $validate->delimiter_teplates(
	report => 1,
	current_input => 1,
  );

Options:

=over 4

=item report

Generate humanly readable report with single fields

=item current_input

Report just current_input and not accumulated data

=back

=cut

sub delimiters_templates {
	my $self = shift;

	my $args = {@_};

	my $t = $self->{_accumulated_delimiters_templates};
	$t = $self->{_delimiters_templates} if ( $args->{current_input} );

	my $log = $self->_get_logger;

	unless ($t) {
		$log->error("called without delimiters");
		return;
	}

	my $out;

	foreach my $f (sort { $a <=> $b } keys %$t) {
		$out .= "$f\n" if ( $args->{report} );
		foreach my $template (sort { $a cmp $b } keys %{ $t->{$f} }) {
			my $count = $t->{$f}->{$template};
			$out .= 
				( $count ? "" : "# " ) .
				( $args->{report} ? "" : "$f" ) .
				"\t$count\t$template\n";
		}
	}

	return $out;
}

=head2 save_delimiters_templates

Save accumulated delimiter templates

  $validator->save_delimiters_template( '/path/to/validate/delimiters' );

=cut

sub save_delimiters_templates {
	my $self = shift;

	my $path = shift;
	$path ||= $self->{delimiters_path};

	my $log = $self->_get_logger;

	$log->logdie("need path") unless ( $path );


	if ( ! $self->{_accumulated_delimiters_templates} ) {
		$log->error('no _accumulated_delimiters_templates found, reset');
		$self->reset;
	}

	if ( $self->{_delimiters_templates} ) {
		$log->error('found _delimiters_templates, calling reset');
		$self->reset;
	}

	$path .= '.new' if ( -e $path );

	open(my $d, '>', $path) || $log->fatal("can't open $path: $!");
	print $d $self->delimiters_templates;
	close($d);

	$log->info("new delimiters templates saved to $path");
}

=head1 AUTHOR

Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>

=head1 COPYRIGHT & LICENSE

Copyright 2006 Dobrica Pavlinusic, All Rights Reserved.

This program is free software; you can redistribute it and/or modify it
under the same terms as Perl itself.

=cut

1; # End of WebPAC::Validate
