use warnings;
use strict;
-use WWW::Mechanize;
use MARC::Record;
-use File::Slurp;
+use Data::Dump qw/dump/;
-binmode STDOUT, ':utf8';
+use base 'Scraper';
my $cobiss_marc21 = {
'010' => { a => [ '020', 'a' ] },
},
205 => { a => [ 250 , 'a' ] },
210 => {
- a => [ 250 , 'a' ],
+ a => [ 260 , 'a' ],
c => [ 260 , 'b' ],
d => [ 260 , 'c' ],
},
},
};
-our $mech = WWW::Mechanize->new();
-our $hits;
-
sub diag {
print "# ", @_, $/;
}
# Koha Z39.50 query:
#
# Bib-1 @and @and @and @and @and @and @and @or
-# @attr 1=8 isbn-issn
-# @attr 1=7 isbn-issn
# @attr 1=4 title
+# @attr 1=7 isbn
+# @attr 1=8 issn
# @attr 1=1003 author
# @attr 1=16 dewey
# @attr 1=21 subject-holding
# @attr 1=1007 standard-id
# @attr 1=1016 any
-our $usemap = {
- 8 => 'BN', # FIXME check
- 7 => 'SN', # FIXME check
+
+# AU= Autor - osoba
+# CB= Autor - korporacija
+# CL= Zbirka
+# CP= Mesto sast./dod. nazivu korp.
+# PP= Mesto izdanja
+# PU= Izdavač
+# PY= Godina izdanja
+# P2= Zaključna godina izdanja
+# TI= Naslov
+# TO= Naslov originala
+# BN= ISBN
+# SN= ISSN uz članak
+# SP= ISSN
+# PN= Predmetna odrednica - lično ime
+# CS= Predm. odred. - naziv korporacije
+# DU= Slobodno oblikovane predm. odred.
+# SU= Predmetne odrednice - sve
+# AC= Kod za vrstu autorstva
+# CC= Kod za vrstu sadržaja
+# CO= Zemlja/regija izdavanja
+# FC= Šifra organizacije
+# LA= Jezik teksta
+# LC= Kod za književni oblik
+# LO= Jezik izvornog dela
+# TA= Kod za predviđene korisnike
+# TD= Tipologija dok./dela
+# UC= UDK za pretraživanje
+# KW= Ključne reči
+
+sub usemap {{
+ 7 => 'BN', # FIXME check
+ 8 => 'SP', # FIXME check
4 => 'TI',
- 1003 => 'TI',
+ 1003 => 'AU',
16 => 'CU',
21 => 'SU',
# 12 => '',
# 1007 => '',
# 1016 => '',
-};
-
-sub usemap {
- my $f = shift || die;
- $usemap->{$f};
-}
+}};
sub search {
my ( $self, $query ) = @_;
die "need query" unless defined $query;
- my $url = 'http://cobiss.izum.si/scripts/cobiss?ukaz=GETID&lani=en';
+# my $url = 'http://cobiss.izum.si/scripts/cobiss?ukaz=GETID&lani=en';
+ my $url = 'http://www.cobiss.ba/scripts/cobiss?ukaz=GETID&lani=en';
-diag "# get $url";
+diag "get $url";
+ my $mech = $self->{mech} || die "no mech?";
+
+ my $hits;
$mech->get( $url );
-diag "# got session";
+diag "got session";
$mech->follow_link( text_regex => qr/union/ );
-diag "# switch to advanced form (select)";
+diag "switch to advanced form (select)";
$mech->follow_link( url_regex => qr/mode=3/ );
-diag "# submit search $query";
+diag "submit search $query";
$mech->submit_form(
fields => {
$hits = 0;
if ( $mech->content =~ m{hits:\s*<b>\s*(\d+)\s*</b>}s ) {
- $hits = $1;
+ $self->{hits} = $hits = $1;
} else {
diag "get't find results in ", $mech->content;
return;
}
-diag "# got $hits results, get first one";
+diag "got $hits results, get first one";
$mech->follow_link( url_regex => qr/ukaz=DISP/ );
-diag "# in COMARC format";
+diag "in COMARC format";
$mech->follow_link( url_regex => qr/fmt=13/ );
+
+ return $hits;
}
-sub fetch_marc {
- my ($self) = @_;
+sub next_marc {
+ my ($self,$format) = @_;
+
+ my $mech = $self->{mech} || die "no mech?";
+
+ $format ||= 'unimarc';
+
+ die "unknown format: $format" unless $format =~ m{(uni|us)marc};
my $comarc;
- if ( $mech->content =~ m{<pre>\s*(.+?(\d+\.)\s+ID=(\d+).+?)\s*</pre>}s ) {
+ if ( $mech->content =~ m{<pre>\s*(.+?(\d+)\.\s+ID=(\d+).+?)\s*</pre>}s ) {
- my $comarc = $1;
+ my $markup = $1;
my $nr = $2;
my $id = $3;
-diag "# fetch_marc $nr [$id]";
+diag "fetch $nr [$id] $format";
- $comarc =~ s{</?b>}{}gs;
- $comarc =~ s{<font[^>]*>}{<s>}gs;
- $comarc =~ s{</font>}{<e>}gs;
+ $markup =~ s{</?b>}{}gs;
+ $markup =~ s{<font[^>]*>}{<s>}gs;
+ $markup =~ s{</font>}{<e>}gs;
- write_file "comarc/$id", $comarc;
+ $markup =~ s/[\r\n]+\s{5}//gs; # join continuation lines
- print $comarc;
+ $self->save_marc( "$id.xml", $markup );
my $marc = MARC::Record->new;
+ my $comarc = MARC::Record->new;
- foreach my $line ( split(/[\r\n]+/, $comarc) ) {
- our @f;
+ foreach my $line ( split(/[\r\n]+/, $markup) ) {
if ( $line !~ s{^(\d\d\d)([01 ])([01 ])}{} ) {
diag "SKIP: $line";
} else {
$line .= "<eol>";
- @f = ( $1, $2, $3 );
- sub sf { push @f, @_; }
- $line =~ s{<s>(\w)<e>([^<]+)\s*}{sf($1, $2)}ges;
- diag "# f:", join('|', @f), " left: |$line|";
- $marc->add_fields( @f );
+ my ( $f, $i1, $i2 ) = ( $1, $2, $3 );
+
+ our $marc_map = undef;
+ our $comarc_map = undef;
+ our $ignored = undef;
+
+ sub sf_parse {
+ my ($f,$sf,$v) = @_;
+
+ $v =~ s/\s+$//;
+
+ push @{ $comarc_map->{ $f } }, ( $sf, $v );
+ if ( my $m = $cobiss_marc21->{$f}->{$sf} ) {
+ push @{ $marc_map->{ $m->[0] } }, ( $m->[1], $v );
+ } else {
+ $ignored->{$f}++;
+ }
+ return ''; # fix warning
+ }
+ my $l = $line;
+ $l =~ s{<s>(\w)<e>([^<]+)}{sf_parse($f,$1, $2)}ges;
+
+ diag "[$format] $line -> ",dump( $comarc_map, $marc_map ) if $comarc_map;
+
+ foreach my $f ( keys %$comarc_map ) {
+ $comarc->add_fields( $f, $i1, $i2, @{ $comarc_map->{$f} } );
+ }
+
+ foreach my $f ( keys %$marc_map ) {
+ $marc->add_fields( $f, $i1, $i2, @{ $marc_map->{$f} } );
+ }
}
}
- open(my $out, '>:utf8', "marc/$id");
- print $out $marc->as_usmarc;
- close($out);
-
+ $self->save_marc( "$id.marc", $marc->as_usmarc );
+ $self->save_marc( "$id.unimarc", $comarc->as_usmarc );
diag $marc->as_formatted;
- return $marc->as_usmarc;
+ if ( $nr < $self->{hits} ) {
+ warn "# fetch next result";
+ $nr++;
+ $mech->follow_link( url_regex => qr/rec=$nr/ );
+ } else {
+ warn "# no more results";
+ }
+
+ return $id;
} else {
die "can't fetch COMARC format from ", $mech->content;
}
projects / Biblio-Z3950.git / blobdiff