X-Git-Url: http://git.rot13.org/?p=crolist2marc;a=blobdiff_plain;f=crolist2marc.pl;h=39eb02885cb43f6a7ead227c9586fa1e18523c0a;hp=a233047ccec10e086e9197d4596d475810572a7c;hb=aa91fb3e8dfa676159100b20c0374cca84ccd886;hpb=b3605e283a79f810d93898e043abe699d0588d68

diff --git a/crolist2marc.pl b/crolist2marc.pl
index a233047..39eb028 100755
--- a/crolist2marc.pl
+++ b/crolist2marc.pl
@@ -5,33 +5,132 @@ use autodie;
 
 use MARC::Record;
 use Data::Dump qw(dump);
+use utf8;
 
 my $data;
 
-my $lines = 0;
+sub csv_file {
+	my ($file,$parse) = @_;
 
-open(my $fh, '<', 'TEKTAG.csv');
-my $h = <$fh>; # header
-while(<$fh>) {
-	chomp;
-	my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $textkey, $textres ) = split(/,/,$_);
+	print STDERR "# reading $file ";
+	my $lines = 1;
 
-	$tagno ||= 0;
-	$sfino ||= 0;
+	open(my $fh, '<', $file);
+	my $h = <$fh>; # header
+	while(<$fh>) {
+		chomp;
+		my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text ) = $parse->($_);
+		$lines++;
 
-	$id .= " " if length $id < 2;
-	$id .= " " if length $id < 2;
-	my ($i1, $i2) = split(//, $id, 2);
+		if ( ! $text ) {
+			print STDERR "\nSKIP $file +$lines [$_] " if $idsl;
+			next;
+		}
 
-	$sfi =~ s/^\$// || die "can't fix subfield [$sfi]";
+		$tagno ||= 0;
+		$sfino ||= 0;
 
-	$data->{$idsl}->{$tag}->[ $tagno ]->[ 0 ] = $i1;
-	$data->{$idsl}->{$tag}->[ $tagno ]->[ 1 ] = $i2;
-	$data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 2 ] = $sfi;
-	$data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 3 ] = $textkey . $textres;
+		$id .= " " if length $id < 2;
+		$id .= " " if length $id < 2;
+		my ($i1, $i2) = split(//, $id, 2);
 
-#	last if $lines++ > 5000;
+		$sfi =~ s/^\$// || die "can't fix subfield [$sfi]";
+
+		$text =~ tr/^~]}\\|[{@`/ČčĆćĐ𩹮ž/; # CROASCII (YUS|HRN) B1.002:1982
+
+		$data->{$idsl}->{$tag}->[ $tagno ]->[ 0 ] = $i1;
+		$data->{$idsl}->{$tag}->[ $tagno ]->[ 1 ] = $i2;
+		$data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 2 ] = $sfi;
+		$data->{$idsl}->{$tag}->[ $tagno ]->[ ( $sfino * 2 ) + 3 ] = $text;
+
+		print STDERR "$lines " if $lines % 1000 == 0;
+	}
+	print STDERR "\n";
 }
 
-warn dump($data);
+csv_file( 'tsv/TEKTAG.csv', sub {
+	my $line = shift;
+
+	my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $textkey, $textres ) = split(/\t/,$_);
+
+	my $text = $textkey . $textres; # FIXME fix CAPITAL letters in $textkey
+
+	return ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text );
+});
+
+csv_file( 'tsv/LONTAG.csv', sub {
+	my $line = shift;
+
+	my ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text ) = split(/\t/,$_, 7);
+
+	$text =~ s/\t+$//;
+	$text =~ s/\t/\n/g; # join OPIS[1-11]
+
+	return ( $idsl, $tag, $tagno, $id, $sfi, $sfino, $text );
+});
+
+csv_file( 'tsv/IDNTAG.csv', sub {
+	my $line = shift;
+
+	my ( $idsl, $tag, $STSL, $KZVS, $BIBRAZ, $HIRAZ, $KPS, $OKO ) = split(/\t/,$_);
+
+	my @leader;
+	$leader[5] = $STSL;
+	$leader[6] = $KZVS;
+	$leader[7] = $BIBRAZ;
+	$leader[8] = $HIRAZ;
+	$leader[17] = $KPS;
+	$leader[18] = $OKO;
+
+	$leader[23] = ' '; # last char;
+
+	my $full = join('', map { defined $_ ? $_ : ' ' } @leader);
+	$data->{$idsl}->{'leader'} = $full; 
+	return;
+});
+
+csv_file( 'tsv/OBRTAG.csv', sub {
+	my $line = shift;
+
+	my ( $IDSL, $TAG, $TAGNO, $ID, $SFI, $SFINO, $CODINF ) = split(/\t/,$_);
+	return ( $IDSL, $TAG, $TAGNO, $ID, $SFI, $SFINO, $CODINF );
+});
+
+print STDERR "\n# getting all ids ";
+my @ids = sort keys %$data;
+print STDERR scalar(@ids), " found\n";
+
+my $marc_file = 'liberated.marc';
+open(my $marc_fh, '>:encoding(UTF-8)', $marc_file);
+my $number = 0;
+
+foreach my $id ( @ids ) {
+	my $rec = MARC::Record->new;
+	$rec->encoding( 'UTF-8' );
+	$rec->add_fields( [ '001', $id ] );
+
+	foreach my $field ( sort keys %{ $data->{$id} } ) {
+		if ( $field eq 'leader' ) {
+			$rec->leader( $data->{$id}->{$field} );
+			next;
+		}
+		foreach my $arr ( @{ $data->{$id}->{$field} } ) {
+			if ( ! $arr ) {
+#				print STDERR "SKIPPED $id $field ",dump( $data->{$id}->{$field} ), "\n";
+				next;
+			}
+			$arr = [ $arr->[3] ] if ( $field < 010 ); # control fields don't have idicators or subfields
+			$rec->add_fields( $field, @$arr );
+		}
+	}
+
+	#print $rec->as_formatted;
+	#print "# $id ",dump($data->{$id});
+
+	print $marc_fh $rec->as_usmarc;
+	$number++;
+	print "$number " if $number % 1000 == 0;
+}
 
+close($marc_fh);
+print "$marc_file ",-s $marc_file, " bytes\n";