#!/usr/bin/perl

use warnings;
use strict;
use autodie;

# apt install libtext-csv-perl libstring-similarity-perl
use Text::CSV;
use Data::Dump qw(dump);
use String::Similarity;
use Storable;

my $debug = $ENV{DEBUG};

my @files = qw( 1.csv 2.csv 3.csv 4.csv );

our $stat;
my $keys;

sub duplicate {
	my @for = @_;
	my $use;
	$use->{$_}++ foreach ( map { @{ $stat->{_}->{$_} } } @for );
	my $duplicate = grep { $use->{$_} > 1 } keys %$use;
	print "XXX use @for ",dump($use),$/ if $debug && $duplicate;
	return $duplicate;
}

sub candidates {
	my ( $num, $key_id, $limit_sim ) = @_;
	$limit_sim //= $ENV{LIMIT};
	$limit_sim //= 0.9;

	my @candidates;
	foreach my $key ( sort keys %{ $keys->{ $key_id } } ) {
		next if $key eq $num; # XXX don't return key from input
		my $s = similarity $num, $key, $limit_sim;
		#warn "# $num $key $s\n";
		if ($s > $limit_sim ) {
			if ( exists $stat->{_}->{"$key-$key_id"} ) {
				my $use;
				$use->{$_}++ foreach (
					@{ $stat->{_}->{"$num-$key_id"} },
					@{ $stat->{_}->{"$key-$key_id"} },
				);
				#print "XXX use $num $key ",dump($use);
				my $duplicate = grep { $use->{$_} > 1 } keys %$use;
				if ( $duplicate ) {
					print "XXX $limit_sim suggest duplicate  $num $key SKIP duplicate ",dump($use), $/ if $debug;
					$stat->{suggest}->{duplicate}++;
				} else {
					push @candidates, { key => $key, s => $s };
				}
			} else {
				print "XXX $limit_sim candidates $key missing\n" if $debug;
			}
		}
	}
=for description
	for limit 0.6 to work, we need to prefer longer results over shorter ones:
	MERGE A0246A 3078-8326 (
	  { key => "A4065A", s => 0.666666666666667 },
	  { key => "ANDREJA0246A", s => 0.666666666666667 },
  ) val=[3]
  .++ A0246A-3078-8326 1 .++ A0246A-3078-8326 2 result val=[3, 1, 2] result_elements=3
=cut

	if ( $#candidates > 0 ) {
		#print "XXX candidates before = ",dump( \@candidates ),$/;
		@candidates = sort {
			$b->{s} <=> $a->{s}	# hi -> low
			or length($b->{key}) <=> length($a->{key})
		} @candidates;
		#print "XXX candidates after  = ",dump( \@candidates ),$/;
	}
	return @candidates;
}

my $keys_file = 'keys.storable';
if ( -e $keys_file ) {
	#$keys = retrieve($keys_file) or die "$keys_file: $!";
	print "LOAD $keys_file", scalar keys %$keys, "\n";
}

foreach my $nr ( 1 .. 4 ) {
	my $file = "$nr.csv";
	warn "# $file\n";

	my $csv = Text::CSV->new ({ binary => 1, auto_diag => 1 });
	open my $fh, "<:encoding(utf8)", $file or die "$file: $!";
	while (my $row = $csv->getline ($fh)) {
		$stat->{lines}->{$nr}++;
		$stat->{ $file }->{lines}++;
		$row->[1] =~ s/\D+//g && $stat->{corrupt}->{skola}->{$nr}++ && print 'c1';
		$row->[2] =~ s/\D+//g && $stat->{corrupt}->{razred}->{$nr}++ && print 'c2';
		my $id = join('-',
			uc $row->[0],
			$row->[1],
			$row->[2],
		);

		my $num = uc $row->[0];
		if ( length $num < 3 ) {
			print "IGNORE $nr ",dump($row->[ 0 .. 5 ]),$/;
			$stat->{ignore}->{$nr}++;
			next;
		}

		my $key_id = $row->[1] . '-' . $row->[2];

		$stat->{A_key_id}->{$key_id}->{$nr}++;

		$keys->{ $key_id }->{ $num }++;

		$stat->{exists}->{$nr}++ if exists $stat->{_}->{ $id };

		push @{ $stat->{_}->{ $id } }, $nr;
	}
	close $fh;

}




my $first = 1;
# 0.9 - 0.7 -- 0.6 is too lax
foreach my $limit ( 0.7, 0.6 ) {
warn "XXX limit $limit\n";

print "# total = ",scalar keys %{ $stat->{_} }, $/;
foreach my $id ( sort keys %{ $stat->{_} } ) {
	my @val;
	my $v = $stat->{_}->{$id};
	if ( defined $v && ref $v eq 'ARRAY' ) {
		@val = @$v;
	} else {
		#print "SKIP[$id]";
		next;
	}
	my $u;
	$u->{$_}++ foreach @val;
	my @u_v = sort keys %$u;
	if ( $first == 1 ) {
		$stat->{A_count}->{ scalar @val }++;
		$stat->{A_count_total}++;

		#$stat->{A_count_val_dup}->{ join(' ', @val) }++; # with duplicates

		$stat->{A_count_val}->{ join(' ', @u_v ) }++; # without duplicates
	}

	if ( $#u_v < 3 ) { # single, double
		my ( $num, $key_id ) = split(/-/,$id,2);
		my @candidates = candidates $num => $key_id, $limit; #, 0.7; # XXX 0.9 too high, 0.8 better, 0.7 too lax
		if ( @candidates ) {
			print "MERGE ",scalar @candidates, " $limit $num $key_id ", dump( @candidates ), ' val=', dump( \@val ), $/;
			my @keys = map { $_->{key} } @candidates;
			my  $m_id = $id; # "$keys[0]-$key_id";
			foreach my $i ( 0 .. $#keys ) {
				my  $id = "$keys[$i]-$key_id";
				if ( ! exists $stat->{_}->{$id} ) {
			       		print "ERROR: $num $key_id can't find $i $id";
				}

				# XXX I298O-4743-7996
				if ( duplicate( $m_id => $id ) ) {
					print "XXX duplicate2 $m_id $id\n";
					$stat->{duplicate2}++;
					next;

				}

				my $o = delete $stat->{_}->{$id};
			       	warn "can't find $id" if ! $o;
				foreach my $val ( @$o ) {
					print '.';
					push @{ $stat->{_}->{ $m_id } }, $val;
					print "++ $m_id $val ";
					$stat->{merge_val}->{$val}++;
				}
				print "result val=",dump( $stat->{_}->{ $m_id } ), " result_elements=", scalar @{ $stat->{_}->{ $m_id } }, $/;
			}
			$stat->{merge}++;
		}

	}
}

print "# total after merge $limit = ",scalar keys %{ $stat->{_} }, $/;

foreach my $id ( sort keys %{ $stat->{_} } ) {
	my @val; # = @{ $stat->{_}->{$id} };
	my $v = $stat->{_}->{$id};
	if ( defined $v && ref $v eq 'ARRAY' ) {
		@val = @$v;
	} else {
		print "SKIP[$id]";
		next;
	}
	$stat->{"B${limit}_count"}->{ scalar @val }++;
	$stat->{"B${limit}_count_total"}++;

	#$stat->{"B${limit}_count_val_dup"}->{ join(' ', @val) }++; # with duplicates

	my $u;
	$u->{$_}++ foreach @val;
	my @u_v = sort keys %$u;
	$stat->{"B${limit}_count_val"}->{ join(' ', @u_v ) }++; # without duplicates
}

$first++;
} # for $limit

print "# stat = ",dump( $stat );
#print "# keys = ",dump( $keys );


store $keys, $keys_file;

