my $data;
-my $lines = 0;
+sub csv_file {
+ my ($file,$parse) = @_;
-open(my $fh, '<', 'TEKTAG.csv');
-my $h = <$fh>; # header
-while(<$fh>) {
- chomp;
- my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $textkey, $textres ) = split(/,/,$_);
+ print STDERR "# reading $file ";
+ my $lines = 0;
- $tagno ||= 0;
- $sfino ||= 0;
+ open(my $fh, '<', $file);
+ my $h = <$fh>; # header
+ while(<$fh>) {
+ chomp;
+ my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text ) = $parse->($_);
+ $lines++;
- $id .= " " if length $id < 2;
- $id .= " " if length $id < 2;
- my ($i1, $i2) = split(//, $id, 2);
+ if ( ! $text ) {
+ print STDERR "\nSKIP $lines [$_] ";
+ next;
+ }
- $sfi =~ s/^\$// || die "can't fix subfield [$sfi]";
+ $tagno ||= 0;
+ $sfino ||= 0;
- my $text = $textkey . $textres; # FIXME fix CAPITAL letters in $textkey
- $text =~ tr/^~]}\|[{@`/ČčĆćĐ𩹮ž/; # CROASCII (YUS|HRN) B1.002:1982
+ $id .= " " if length $id < 2;
+ $id .= " " if length $id < 2;
+ my ($i1, $i2) = split(//, $id, 2);
+
+ $sfi =~ s/^\$// || die "can't fix subfield [$sfi]";
+
+ $text =~ tr/^~]}\|[{@`/ČčĆćĐ𩹮ž/; # CROASCII (YUS|HRN) B1.002:1982
- $data->{$idsl}->{$tag}->[ $tagno ]->[ 0 ] = $i1;
- $data->{$idsl}->{$tag}->[ $tagno ]->[ 1 ] = $i2;
- $data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 2 ] = $sfi;
- $data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 3 ] = $text;
+ $data->{$idsl}->{$tag}->[ $tagno ]->[ 0 ] = $i1;
+ $data->{$idsl}->{$tag}->[ $tagno ]->[ 1 ] = $i2;
+ $data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 2 ] = $sfi;
+ $data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 3 ] = $text;
-# last if $lines++ > 5000;
+ print STDERR "$lines " if $lines % 1000 == 0;
+ }
+ print STDERR "\n";
}
-warn dump($data);
+csv_file( 'TEKTAG.csv', sub {
+ my $line = shift;
+
+ my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $textkey, $textres ) = split(/,/,$_);
+
+ my $text = $textkey . $textres; # FIXME fix CAPITAL letters in $textkey
+
+ return ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text );
+});
+
+csv_file( 'LONTAG.csv', sub {
+ my $line = shift;
+
+ my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text ) = split(/;/,$_, 7);
+
+ $text =~ s/;+$//;
+ $text =~ s/;/\n/g; # join OPIS[1-11]
+
+ return ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text );
+});
+
+print STDERR "\n# getting all ids ";
+my @ids = keys %$data;
+print STDERR scalar(@ids), " found\n";
+foreach my $id ( @ids ) {
+ print "# $id ",dump($data->{$id});
+}