ignore \N fields (NULL)

[webpac2] / bin / isi-merge.pl

diff --git a/bin/isi-merge.pl b/bin/isi-merge.pl
index aca4d81..64fabab 100755 (executable)
--- a/bin/isi-merge.pl
+++ b/bin/isi-merge.pl
@@ -4,12 +4,18 @@ use warnings;
 use strict;
 use autodie;
 use Digest::MD5 qw(md5_hex);
+use Digest::SHA1 qw(sha1_hex);
 use Data::Dump qw(dump);
 
+my $use_sha1 = $ENV{SHA1} || 1;
+
 my @files = @ARGV;
 @files = glob '/tmp/isi.*-*.txt' unless @files;
 
 my $path = '/tmp/isi.full.txt';
+
+warn "# ", $#files + 1, " files to $path sha:$use_sha1\n";
+
 open(my $out_fh, '>', $path);
 print $out_fh "FN ISI Export Format\nVR 1.0\n";
 
@@ -25,7 +31,7 @@ foreach my $file ( sort {
        my $b_r = $1 if $b =~ m{(\d+)-\d+};
        $a_r <=> $b_r;
 } @files ) {
-       warn $file;
+       print STDERR $file;
 
        push @{ $report->{files} }, $file;
 
@@ -37,20 +43,22 @@ foreach my $file ( sort {
                        $rec .= $_;
                } else {
                        $nr++;
-                       my $digest = md5_hex $rec;
+                       my $digest = $use_sha1 ? sha1_hex $rec : md5_hex $rec;
                        if ( my $times = $md5->{$digest} ) {
-                               warn "dumplicate $nr $digest $times\n";
+                               print STDERR 'd';
                                $report->{file}->{$file}->{duplicates}++;
                        } else {
                                print $out_fh $rec . $_;
                                $report->{file}->{$file}->{records}++;
                                $report->{total_records}++;
+                               print STDERR '.';
                        }
 
                        $md5->{$digest}++;
                        $rec = '';
                }
        }
+       warn "\n";
 }
 
 print $out_fh "EF\n";