X-Git-Url: http://git.rot13.org/?p=Biblio-Z3950.git;a=blobdiff_plain;f=COBISS.pm;h=81314eabdfac25c97ed59842c38b379d42b74ead;hp=1de03c322eef362b51ed94655e09b37b086b1192;hb=HEAD;hpb=5e69d504e9b9535fb13c5311a9aab314f2d6bfdd

diff --git a/COBISS.pm b/COBISS.pm
index 1de03c3..81314ea 100644
--- a/COBISS.pm
+++ b/COBISS.pm
@@ -3,11 +3,10 @@ package COBISS;
 use warnings;
 use strict;
 
-use WWW::Mechanize;
 use MARC::Record;
-use File::Slurp;
+use Data::Dump qw/dump/;
 
-binmode STDOUT, ':utf8';
+use base 'Scraper';
 
 my $cobiss_marc21 = {
 	'010' => { a => [ '020', 'a' ] },
@@ -17,7 +16,7 @@ my $cobiss_marc21 = {
 	},
 	 205  => { a => [  250 , 'a' ] },
 	 210  => {
-		a => [  250 , 'a' ],
+		a => [  260 , 'a' ],
 		c => [  260 , 'b' ],
 		d => [  260 , 'c' ],
 	},
@@ -31,9 +30,6 @@ my $cobiss_marc21 = {
 	},
 };
 
-our $mech = WWW::Mechanize->new();
-our $hits;
-
 sub diag {
 	print "# ", @_, $/;
 }
@@ -41,9 +37,9 @@ sub diag {
 # Koha Z39.50 query:
 #
 # Bib-1 @and @and @and @and @and @and @and @or
-# @attr 1=8 isbn-issn 
-# @attr 1=7 isbn-issn 
 # @attr 1=4 title 
+# @attr 1=7 isbn
+# @attr 1=8 issn 
 # @attr 1=1003 author 
 # @attr 1=16 dewey 
 # @attr 1=21 subject-holding 
@@ -51,46 +47,73 @@ sub diag {
 # @attr 1=1007 standard-id 
 # @attr 1=1016 any
 
-our $usemap = {
-	8		=> 'BN',	# FIXME check
-	7		=> 'SN',	# FIXME check
+
+# AU=	Autor - osoba
+# CB=	Autor - korporacija
+# CL=	Zbirka
+# CP=	Mesto sast./dod. nazivu korp.
+# PP=	Mesto izdanja
+# PU=	Izdavač
+# PY=	Godina izdanja
+# P2=	Zaključna godina izdanja
+# TI=	Naslov
+# TO=	Naslov originala
+# BN=	ISBN
+# SN=	ISSN uz članak
+# SP=	ISSN
+# PN=	Predmetna odrednica - lično ime
+# CS=	Predm. odred. - naziv korporacije
+# DU=	Slobodno oblikovane predm. odred.
+# SU=	Predmetne odrednice - sve
+# AC=	Kod za vrstu autorstva
+# CC=	Kod za vrstu sadržaja
+# CO=	Zemlja/regija izdavanja
+# FC=	Å ifra organizacije
+# LA=	Jezik teksta
+# LC=	Kod za književni oblik
+# LO=	Jezik izvornog dela
+# TA=	Kod za predviđene korisnike
+# TD=	Tipologija dok./dela
+# UC=	UDK za pretraživanje
+# KW=	Ključne reči
+
+sub usemap {{
+	7		=> 'BN',	# FIXME check
+	8		=> 'SP',	# FIXME check
 	4		=> 'TI',
-	1003	=> 'TI',
+	1003	=> 'AU',
 	16		=> 'CU',
 	21		=> 'SU',
 #	12		=> '',
 #	1007	=> '',
 #	1016	=> '',
 
-};
-
-sub usemap {
-	my $f = shift || die;
-	$usemap->{$f};
-}
+}};
 
 sub search {
 	my ( $self, $query ) = @_;
 
 	die "need query" unless defined $query;
 
-	my $url = 'http://cobiss.izum.si/scripts/cobiss?ukaz=GETID&lani=en';
+#	my $url = 'http://cobiss.izum.si/scripts/cobiss?ukaz=GETID&lani=en';
+	my $url = 'http://www.cobiss.ba/scripts/cobiss?ukaz=GETID&lani=en';
+
+diag "get $url";
 
-diag "# get $url";
+	my $mech = $self->{mech} || die "no mech?";
 
+	my $hits;
 	$mech->get( $url );
 
-diag "# got session";
+diag "got session";
 
 	$mech->follow_link( text_regex => qr/union/ );
 
-diag "# switch to advanced form (select)";
-
-diag $mech->content;
+diag "switch to advanced form (select)";
 
 	$mech->follow_link( url_regex => qr/mode=3/ );
 
-diag "# submit search $query";
+diag "submit search $query";
 
 	$mech->submit_form(
 		fields => {
@@ -100,68 +123,108 @@ diag "# submit search $query";
 
 	$hits = 0;
 	if ( $mech->content =~ m{hits:\s*<b>\s*(\d+)\s*</b>}s ) {
-		$hits = $1;
+		$self->{hits} = $hits = $1;
 	} else {
 		diag "get't find results in ", $mech->content;
 		return;
 	}
 
-diag "# got $hits results, get first one";
+diag "got $hits results, get first one";
 
 	$mech->follow_link( url_regex => qr/ukaz=DISP/ );
 
-diag "# in COMARC format";
+diag "in COMARC format";
 
 	$mech->follow_link( url_regex => qr/fmt=13/ );
+
+	return $hits;
 }
 
 
-sub fetch_marc {
-	my ($self) = @_;
+sub next_marc {
+	my ($self,$format) = @_;
+
+	my $mech = $self->{mech} || die "no mech?";
+
+	$format ||= 'unimarc';
+
+	die "unknown format: $format" unless $format =~ m{(uni|us)marc};
 
 	my $comarc;
 
-	if ( $mech->content =~ m{<pre>\s*(.+?(\d+\.)\s+ID=(\d+).+?)\s*</pre>}s ) {
+	if ( $mech->content =~ m{<pre>\s*(.+?(\d+)\.\s+ID=(\d+).+?)\s*</pre>}s ) {
 
-		my $comarc = $1;
+		my $markup = $1;
 		my $nr = $2;
 		my $id = $3;
 
-diag "# fetch_marc $nr [$id]";
+diag "fetch $nr [$id] $format";
 
-		$comarc =~ s{</?b>}{}gs;
-		$comarc =~ s{<font[^>]*>}{<s>}gs;
-		$comarc =~ s{</font>}{<e>}gs;
+		$markup =~ s{</?b>}{}gs;
+		$markup =~ s{<font[^>]*>}{<s>}gs;
+		$markup =~ s{</font>}{<e>}gs;
 
-		write_file "comarc/$id", $comarc;
+		$markup =~ s/[\r\n]+\s{5}//gs; # join continuation lines
 
-		print $comarc;
+		$self->save_marc( "$id.xml", $markup );
 
 		my $marc = MARC::Record->new;
+		my $comarc = MARC::Record->new;
 
-		foreach my $line ( split(/[\r\n]+/, $comarc) ) {
-			our @f;
+		foreach my $line ( split(/[\r\n]+/, $markup) ) {
 
 			if ( $line !~ s{^(\d\d\d)([01 ])([01 ])}{} ) {
 				diag "SKIP: $line";
 			} else {
 				$line .= "<eol>";
 
-				@f = ( $1, $2, $3 );
-				sub sf { push @f, @_; }
-				$line =~ s{<s>(\w)<e>([^<]+)\s*}{sf($1, $2)}ges;
-				diag "# f:", join('|', @f), " left: |$line|";
-				$marc->add_fields( @f );
+				my ( $f, $i1, $i2 ) = ( $1, $2, $3 );
+
+				our $marc_map = undef;
+				our $comarc_map = undef;
+				our $ignored = undef;
+
+				sub sf_parse {
+					my ($f,$sf,$v) = @_;
+
+					$v =~ s/\s+$//;
+
+					push @{ $comarc_map->{ $f } }, ( $sf, $v );
+					if ( my $m = $cobiss_marc21->{$f}->{$sf} ) {
+						push @{ $marc_map->{ $m->[0] } }, ( $m->[1], $v );
+					} else {
+						$ignored->{$f}++;
+					}
+					return ''; # fix warning
+				}
+				my $l = $line;
+				$l =~ s{<s>(\w)<e>([^<]+)}{sf_parse($f,$1, $2)}ges;
+
+				diag "[$format] $line -> ",dump( $comarc_map, $marc_map ) if $comarc_map;
+
+				foreach my $f ( keys %$comarc_map ) {
+					$comarc->add_fields( $f, $i1, $i2, @{ $comarc_map->{$f} } );
+				}
+
+				foreach my $f ( keys %$marc_map ) {
+					$marc->add_fields( $f, $i1, $i2, @{ $marc_map->{$f} } );
+				}
 			}
 		}
 
-		open(my $out, '>:utf8', "marc/$id");
-		print $out $marc->as_usmarc;
-		close($out);
-
+		$self->save_marc( "$id.marc", $marc->as_usmarc );
+		$self->save_marc( "$id.unimarc", $comarc->as_usmarc );
 		diag $marc->as_formatted;
 
-		return $marc->as_usmarc;
+		if ( $nr < $self->{hits} ) {
+			warn "# fetch next result";
+			$nr++;
+			$mech->follow_link( url_regex => qr/rec=$nr/ );
+		} else {
+			warn "# no more results";
+		}
+
+		return $id;
 	} else {
 		die "can't fetch COMARC format from ", $mech->content;
 	}