parse one more file, extract common parts

diff --git a/crolist2marc.pl b/crolist2marc.pl
index 56a54f2..ef4b056 100755 (executable)
--- a/crolist2marc.pl
+++ b/crolist2marc.pl
@@ -9,33 +9,70 @@ use utf8;
 
 my $data;
 
 
 my $data;
 

-my $lines = 0;
+sub csv_file {
+       my ($file,$parse) = @_;

 
 

-open(my $fh, '<', 'TEKTAG.csv');
-my $h = <$fh>; # header
-while(<$fh>) {
-       chomp;
-       my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $textkey, $textres ) = split(/,/,$_);
+       print STDERR "# reading $file ";
+       my $lines = 0;

 
 

-       $tagno ||= 0;
-       $sfino ||= 0;
+       open(my $fh, '<', $file);
+       my $h = <$fh>; # header
+       while(<$fh>) {
+               chomp;
+               my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text ) = $parse->($_);
+               $lines++;

 
 

-       $id .= " " if length $id < 2;
-       $id .= " " if length $id < 2;
-       my ($i1, $i2) = split(//, $id, 2);
+               if ( ! $text ) {
+                       print STDERR "\nSKIP $lines [$_] ";
+                       next;
+               }

 
 

-       $sfi =~ s/^\$// || die "can't fix subfield [$sfi]";
+               $tagno ||= 0;
+               $sfino ||= 0;

 
 

-       my $text = $textkey . $textres; # FIXME fix CAPITAL letters in $textkey
-       $text =~ tr/^~]}\|[{@`/ČčĆćĐ𩹮ž/; # CROASCII (YUS|HRN) B1.002:1982
+               $id .= " " if length $id < 2;
+               $id .= " " if length $id < 2;
+               my ($i1, $i2) = split(//, $id, 2);
+
+               $sfi =~ s/^\$// || die "can't fix subfield [$sfi]";
+
+               $text =~ tr/^~]}\|[{@`/ČčĆćĐ𩹮ž/; # CROASCII (YUS|HRN) B1.002:1982

 
 

-       $data->{$idsl}->{$tag}->[ $tagno ]->[ 0 ] = $i1;
-       $data->{$idsl}->{$tag}->[ $tagno ]->[ 1 ] = $i2;
-       $data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 2 ] = $sfi;
-       $data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 3 ] = $text;
+               $data->{$idsl}->{$tag}->[ $tagno ]->[ 0 ] = $i1;
+               $data->{$idsl}->{$tag}->[ $tagno ]->[ 1 ] = $i2;
+               $data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 2 ] = $sfi;
+               $data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 3 ] = $text;

 
 

-#      last if $lines++ > 5000;
+               print STDERR "$lines " if $lines % 1000 == 0;
+       }
+       print STDERR "\n";

 }
 
 }
 

-warn dump($data);
+csv_file( 'TEKTAG.csv', sub {
+       my $line = shift;
+
+       my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $textkey, $textres ) = split(/,/,$_);
+
+       my $text = $textkey . $textres; # FIXME fix CAPITAL letters in $textkey
+
+       return ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text );
+});
+
+csv_file( 'LONTAG.csv', sub {
+       my $line = shift;
+
+       my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text ) = split(/;/,$_, 7);
+
+       $text =~ s/;+$//;
+       $text =~ s/;/\n/g; # join OPIS[1-11]
+
+       return ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text );
+});
+
+print STDERR "\n# getting all ids ";
+my @ids = keys %$data;
+print STDERR scalar(@ids), " found\n";
+foreach my $id ( @ids ) {
+       print "# $id ",dump($data->{$id});
+}