#!/usr/bin/perl

use warnings;
use strict;
use autodie;

# apt install libtext-csv-perl libstring-similarity-perl
use Text::CSV;
use Data::Dump qw(dump);
use String::Similarity;
use Storable;

my $debug = $ENV{DEBUG};

my @files = qw( 1.csv 2.csv 3.csv 4.csv );

our $stat;
my $keys;

sub duplicate {
	my @for = @_;
	my $use;
	$use->{$_}++ foreach ( map { @{ $stat->{_}->{$_} } } @for );
	my $duplicate = grep { $use->{$_} > 1 } keys %$use;
	print "XXX use @for ",dump($use),$/ if $debug && $duplicate;
	return $duplicate;
}

sub candidates {
	my ( $num, $key_id, $limit_sim ) = @_;
	$limit_sim //= $ENV{LIMIT};
	$limit_sim //= 0.9;

	my @candidates;
	foreach my $key ( sort keys %{ $keys->{ $key_id } } ) {
		next if $key eq $num; # XXX don't return key from input
		my $s = similarity $num, $key, $limit_sim;
		#warn "# $num $key $s\n";
		if ($s > $limit_sim ) {
			if ( exists $stat->{_}->{"$key-$key_id"} ) {
				my $use;
				$use->{$_}++ foreach (
					@{ $stat->{_}->{"$num-$key_id"} },
					@{ $stat->{_}->{"$key-$key_id"} },
				);
				#print "XXX use $num $key ",dump($use);
				my $duplicate = grep { $use->{$_} > 1 } keys %$use;
				if ( $duplicate ) {
					print "XXX $limit_sim suggest duplicate  $num $key SKIP duplicate ",dump($use), $/ if $debug;
					$stat->{suggest}->{duplicate}++;
				} else {
					push @candidates, { key => $key, s => $s };
				}
			} else {
				print "XXX $limit_sim candidates $key missing\n" if $debug;
			}
		}
	}
=for description
	for limit 0.6 to work, we need to prefer longer results over shorter ones:
	MERGE A0246A 3078-8326 (
	  { key => "A4065A", s => 0.666666666666667 },
	  { key => "ANDREJA0246A", s => 0.666666666666667 },
  ) val=[3]
  .++ A0246A-3078-8326 1 .++ A0246A-3078-8326 2 result val=[3, 1, 2] result_elements=3
=cut

	if ( $#candidates > 0 ) {
		#print "XXX candidates before = ",dump( \@candidates ),$/;
		@candidates = sort {
			$b->{s} <=> $a->{s}	# hi -> low
			or length($b->{key}) <=> length($a->{key})
		} @candidates;
		#print "XXX candidates after  = ",dump( \@candidates ),$/;
	}
	return @candidates;
}

my $keys_file = 'keys.storable';
if ( -e $keys_file ) {
	#$keys = retrieve($keys_file) or die "$keys_file: $!";
	print "LOAD $keys_file", scalar keys %$keys, "\n";
}

our $data;
our @data_headers;


foreach my $val ( 1 .. 4 ) {
	my $file = "$val.csv";
	warn "# $file\n";

	open(my $duplicate_fh, '>', "duplicate-$val.csv");

	my $csv = Text::CSV->new ({ binary => 1, auto_diag => 1 });
	open my $fh, "<:encoding(utf8)", $file or die "$file: $!";
	while (my $row = $csv->getline ($fh)) {
		$stat->{file}->{$file}->{lines}++;
		$stat->{file}->{$file}->{columns}->{ $#$row }++;

		if ( ! exists $data_headers[$val] ) {
			$data_headers[$val] = $row;
			next;
		}

		my $c_id = $row->[0];
		my $c_s = $row->[1];
		my $c_r = $row->[2];
		$row->[0] =~ s/[^\w\d]+//ig && $stat->{file}->{$file}->{corrupt_id}->{$c_id}++ && print 'c0';
		$row->[1] =~ s/\D+//g && $stat->{file}->{$file}->{corrupt_s}->{$c_s}++ && print 'c1';
		$row->[2] =~ s/\D+//g && $stat->{file}->{$file}->{corrupt_r}->{$c_r}++ && print 'c2';
		my $id = join('-',
			uc $row->[0],
			$row->[1],
			$row->[2],
		);

		my $num = uc $row->[0];
		if ( length $num < 3 ) {
			print "IGNORE $val ",dump($row->[ 0 .. 5 ]),$/;
			$stat->{ignore}->{$val}++;
			next;
		}

		my $key_id = $row->[1] . '-' . $row->[2];

		$stat->{A_key_id}->{$key_id}->{$val}++;

		$keys->{ $key_id }->{ $num }++;

		$stat->{exists}->{$val}++ if exists $stat->{_}->{ $id };

		push @{ $stat->{_}->{ $id } }, $val;

		if ( exists $data->{$key_id}->{$num}->{$val} ) {
			$stat->{file}->{$file}->{duplicate_keyid_num}->{$val}++;
			print "DUPLICATE $file $key_id $num $val\n";
			
			my $old = $data->{$key_id}->{$num}->{$val};
			print $duplicate_fh join(',', $file, @$old), "\n";
			print $duplicate_fh join(',', $file, @$row), "\n";
			print $duplicate_fh "\n";
=for diff
			my $diff;
			foreach ( 0 .. $#$row ) {
				if ( $old->[$_] ne $row->[$_] ) {
					$diff->[$_] = [ $old->[$_], $row->[$_] ];
				}
			}
			print "diff = ",dump($diff) if $diff;
			#print "old=", dump( $data->{$key_id}->{$num}->{$val} ), $/;
			#print "new=", dump( $row ), $/;
=cut
		}
		# remove .000000 from values
		$data->{$key_id}->{$num}->{$val} = [ map { s/^(\d+)\.0+$/$1/g; $_; } @$row ];
	}
	close $fh;

}


my $merge_ids;

my $first = 1;
# 0.9 - 0.7 -- 0.6 is too lax
foreach my $limit ( 0.7 ) { #, 0.6 ) {
warn "XXX limit $limit\n";

print "# total = ",scalar keys %{ $stat->{_} }, $/;
foreach my $id ( sort keys %{ $stat->{_} } ) {
	my @val;
	my $v = $stat->{_}->{$id};
	if ( defined $v && ref $v eq 'ARRAY' ) {
		@val = @$v;
	} else {
		#print "SKIP[$id]";
		next;
	}
	my $u;
	$u->{$_}++ foreach @val;
	my @u_v = sort keys %$u;
	if ( $first == 1 ) {
		$stat->{A_count}->{ scalar @val }++;
		$stat->{A_count_total}++;

		#$stat->{A_count_val_dup}->{ join(' ', @val) }++; # with duplicates

		$stat->{A_count_val}->{ join(' ', @u_v ) }++; # without duplicates
	}

	if ( $#u_v < 3 ) { # single, double
		my ( $num, $key_id ) = split(/-/,$id,2);
		my @candidates = candidates $num => $key_id, $limit; #, 0.7; # XXX 0.9 too high, 0.8 better, 0.7 too lax
		if ( @candidates ) {
			print "MERGE ",scalar @candidates, " $limit $num $key_id ", dump( @candidates ), ' val=', dump( \@val ), $/;
			my @keys = map { $_->{key} } @candidates;
			my  $m_id = $id; # "$keys[0]-$key_id";
			foreach my $i ( 0 .. $#keys ) {
				my  $id = "$keys[$i]-$key_id";
				if ( ! exists $stat->{_}->{$id} ) {
			       		print "ERROR: $num $key_id can't find $i $id";
				}

				# XXX I298O-4743-7996
				if ( duplicate( $m_id => $id ) ) {
					print "XXX duplicate2 $m_id $id\n";
					$stat->{duplicate2}++;
					next;

				}

				my $o = delete $stat->{_}->{$id};
				die "FATAL: can't find $id" if ! $o;

				my ( $id_s, $s, $r ) = split('-', $id);
				my $key_s = "$s-$r";

				foreach my $val ( @$o ) {
					print '.';
					push @{ $stat->{_}->{ $m_id } }, $val;
					print "++ $m_id $val ";
					$stat->{merge_val}->{$val}++;


					die "ERROR merge: $val $id $m_id exists",dump( $merge_ids->{$val}->{$key_s}->{$id_s} ) if exists $merge_ids->{$val}->{$key_s}->{$id_s};
					my $m_id_s = (split('-',$m_id,3))[0];
					$merge_ids->{$val}->{$key_s}->{$id_s} = $m_id_s;

					my $o_row = delete $data->{$key_s}->{$id_s}->{$val};
					die "FATAL: $id | $m_id | data $key_s $id_s $val" unless $o_row;
					$data->{$key_s}->{$m_id_s}->{$val} = $o_row;

				}
				my @not_empty = sort keys %{ $data->{$key_s}->{$id_s} };
				die "FATAL: $id_s not empty" if @not_empty;
				delete $data->{$key_s}->{$id_s}; # FIXME check before cleanup

				print "result val=",dump( $stat->{_}->{ $m_id } ), " result_elements=", scalar @{ $stat->{_}->{ $m_id } }, $/;
			}
			$stat->{merge}++;
		}

	}
}

print "# total after merge $limit = ",scalar keys %{ $stat->{_} }, $/;

foreach my $id ( sort keys %{ $stat->{_} } ) {
	my @val; # = @{ $stat->{_}->{$id} };
	my $v = $stat->{_}->{$id};
	if ( defined $v && ref $v eq 'ARRAY' ) {
		@val = @$v;
	} else {
		print "SKIP[$id]";
		next;
	}
	$stat->{"B${limit}_count"}->{ scalar @val }++;
	$stat->{"B${limit}_count_total"}++;

	#$stat->{"B${limit}_count_val_dup"}->{ join(' ', @val) }++; # with duplicates

	my $u;
	$u->{$_}++ foreach @val;
	my @u_v = sort keys %$u;
	$stat->{"B${limit}_count_val"}->{ join(' ', @u_v ) }++; # without duplicates
}

$first++;
} # for $limit

print "# stat = ",dump( $stat );
#print "# keys = ",dump( $keys );

store $keys, $keys_file;

my $merge_file = 'merge.storable';
store $merge_ids, $merge_file;
#print "XXX merge_ids = ", dump($merge_ids);

my $out_file = 'merged.csv';
print "out_file $out_file";
open(my $out_fh, '>', $out_file);
foreach my $val ( 1 .. 4 ) {
	print $out_fh join(',', map { $_ . '_' . $val } @{ $data_headers[$val] });
	print $out_fh ',';
}
print $out_fh "broj_valova";
print $out_fh "\n";

foreach my $key ( sort keys %$data ) {
	print " $key";
	foreach my $id ( sort keys %{ $data->{$key} } ) {
		#print $out_fh "## $id ## ";

		my $broj_valova = 0;

		foreach my $val ( 1 .. 4 ) {
			if ( my $id_data = $data->{$key}->{$id}->{$val} ) {
				print $out_fh join(',', @$id_data);
				$broj_valova++;
			} else {
				print $out_fh ( ',' x $#{ $data_headers[$val] } ) ; # FIXME +1?
			}
			print $out_fh ',';
		}
		print $out_fh $broj_valova;
		print $out_fh "\n";
	}
}
close $out_fh;
print "\n", -s $out_file, " bytes created\n";

