deduplicate records using md5

git-svn-id: svn+ssh://mjesec/home/dpavlin/svn/webpac2/trunk@1284 07558da8-63fa-0310-ba24-9fe276d99e06

diff --git a/bin/isi-merge.pl b/bin/isi-merge.pl
index c7d76e6..aca4d81 100755 (executable)
--- a/bin/isi-merge.pl
+++ b/bin/isi-merge.pl
@@ -3,6 +3,8 @@
 use warnings;
 use strict;
 use autodie;
+use Digest::MD5 qw(md5_hex);
+use Data::Dump qw(dump);
 
 my @files = @ARGV;
 @files = glob '/tmp/isi.*-*.txt' unless @files;
@@ -11,6 +13,13 @@ my $path = '/tmp/isi.full.txt';
 open(my $out_fh, '>', $path);
 print $out_fh "FN ISI Export Format\nVR 1.0\n";
 
+my $rec;
+my $nr = 0;
+
+my $md5;
+
+my $report;
+
 foreach my $file ( sort {
        my $a_r = $1 if $a =~ m{(\d+)-\d+};
        my $b_r = $1 if $b =~ m{(\d+)-\d+};
@@ -18,15 +27,35 @@ foreach my $file ( sort {
 } @files ) {
        warn $file;
 
+       push @{ $report->{files} }, $file;
+
        open(my $fh, '<', $file);
        while(<$fh>) {
                next if m/^(FN|VR|EF)/;
 
-               print $out_fh $_;
+               if ( ! m/^[\r\n]+$/s ) {
+                       $rec .= $_;
+               } else {
+                       $nr++;
+                       my $digest = md5_hex $rec;
+                       if ( my $times = $md5->{$digest} ) {
+                               warn "dumplicate $nr $digest $times\n";
+                               $report->{file}->{$file}->{duplicates}++;
+                       } else {
+                               print $out_fh $rec . $_;
+                               $report->{file}->{$file}->{records}++;
+                               $report->{total_records}++;
+                       }
+
+                       $md5->{$digest}++;
+                       $rec = '';
+               }
        }
 }
 
 print $out_fh "EF\n";
 close $out_fh;
 
-warn "# $path ", -s $path;
+
+warn "# $path ", -s $path, dump $report;
+