use warnings;
use strict;
-use WWW::Mechanize;
use MARC::Record;
use Data::Dump qw/dump/;
-binmode STDOUT, ':utf8';
+use base 'Scraper';
my $cobiss_marc21 = {
'010' => { a => [ '020', 'a' ] },
},
205 => { a => [ 250 , 'a' ] },
210 => {
- a => [ 250 , 'a' ],
- c => [ 250 , 'b' ],
- d => [ 250 , 'c' ],
+ a => [ 260 , 'a' ],
+ c => [ 260 , 'b' ],
+ d => [ 260 , 'c' ],
},
215 => {
a => [ 300 , 'a' ],
},
};
-our $mech = WWW::Mechanize->new();
-our $hits;
-
sub diag {
print "# ", @_, $/;
}
# @attr 1=1007 standard-id
# @attr 1=1016 any
-our $usemap = {
+sub usemap {{
8 => 'BN', # FIXME check
7 => 'SN', # FIXME check
4 => 'TI',
# 1007 => '',
# 1016 => '',
-};
-
-sub usemap {
- my $f = shift || die;
- $usemap->{$f};
-}
+}};
sub search {
my ( $self, $query ) = @_;
diag "get $url";
+ my $mech = $self->{mech} || die "no mech?";
+
+ my $hits;
$mech->get( $url );
diag "got session";
$hits = 0;
if ( $mech->content =~ m{hits:\s*<b>\s*(\d+)\s*</b>}s ) {
- $hits = $1;
+ $self->{hits} = $hits = $1;
} else {
diag "get't find results in ", $mech->content;
return;
diag "in COMARC format";
$mech->follow_link( url_regex => qr/fmt=13/ );
+
+ return $hits;
}
-sub fetch_rec {
+sub next_marc {
my ($self,$format) = @_;
+ my $mech = $self->{mech} || die "no mech?";
+
$format ||= 'unimarc';
die "unknown format: $format" unless $format =~ m{(uni|us)marc};
my $marc = MARC::Record->new;
+ $comarc =~ s/[\r\n]+\s{5}//gs; # join continuation lines
+warn "## comarc join: $comarc\n";
+
foreach my $line ( split(/[\r\n]+/, $comarc) ) {
if ( $line !~ s{^(\d\d\d)([01 ])([01 ])}{} ) {
diag "SKIP: $line";
} else {
- our @f = ( $1, $2, $3 );
$line .= "<eol>";
- if ( $format eq 'unimarc' ) {
+ my ( $f, $i1, $i2 ) = ( $1, $2, $3 );
- diag dump(@f), "line: $line";
- sub sf_uni {
- warn "sf ",dump(@_);
- push @f, @_;
- }
- $line =~ s{<s>(\w)<e>([^<]+)\s*}{sf_uni($1, $2)}ges;
- diag "f:", dump(@f), " left: |$line|";
- $marc->add_fields( @f );
+ our $out = undef;
+ our $ignored = undef;
- } elsif ( $format eq 'usmarc' ) {
+ sub sf_us {
+ my ($format,$f,$sf,$v) = @_;
- my ( $f, $i1, $i2 ) = @f;
+ $v =~ s/\s+$//;
- our $out = {};
-
- sub sf_us {
- my ($f,$sf,$v) = @_;
+ if ( $format =~ m/unimarc/i ) {
+ push @{ $out->{ $f } }, ( $sf, $v );
+ } elsif ( $format =~ m/marc/ ) {
if ( my $m = $cobiss_marc21->{$f}->{$sf} ) {
push @{ $out->{ $m->[0] } }, ( $m->[1], $v );
+ } else {
+ $ignored->{$f}++;
}
- return;
}
- $line =~ s{<s>(\w)<e>([^<]+)\s*}{sf_us($f,$1, $2)}ges;
+ return ''; # fix warning
+ }
+ my $l = $line;
+ $l =~ s{<s>(\w)<e>([^<]+)}{sf_us($format,$f,$1, $2)}ges;
- diag "converted marc21 ",dump( $out );
+ diag "[$format] $line -> ",dump( $out ) if $out;
- foreach my $f ( keys %$out ) {
- $marc->add_fields( $f, $i1, $i2, @{ $out->{$f} } );
- }
+ foreach my $f ( keys %$out ) {
+ $marc->add_fields( $f, $i1, $i2, @{ $out->{$f} } );
}
}
}
- my $path = "marc/$id.$format";
-
- open($out, '>:utf8', $path);
- print $out $marc->as_usmarc;
- close($out);
-
- diag "created $path ", -s $path, " bytes";
-
+ $self->save_marc( "$id.$format", $marc->as_usmarc );
diag $marc->as_formatted;
- $nr++;
- $mech->follow_link( url_regex => qr/rec=$nr/ );
+ if ( $nr < $self->{hits} ) {
+ warn "# fetch next result";
+ $nr++;
+ $mech->follow_link( url_regex => qr/rec=$nr/ );
+ } else {
+ warn "# no more results";
+ }
return $marc->as_usmarc;
} else {