extract common code into Scraper package

[Biblio-Z3950.git] / Aleph.pm

diff --git a/Aleph.pm b/Aleph.pm
index 953fda7..71d67a7 100644 (file)
--- a/Aleph.pm
+++ b/Aleph.pm
@@ -3,11 +3,10 @@ package Aleph;
 use warnings;
 use strict;
 
-use WWW::Mechanize;
 use MARC::Record;
 use Data::Dump qw/dump/;
 
-binmode STDOUT, ':utf8';
+use base 'Scraper';
 
 our $mech = WWW::Mechanize->new();
 our $hits;
@@ -53,9 +52,7 @@ sub diag {
 # WGA - Riječi u geografskim odrednicama 
 # WYR - Godina izdavanja
 
-our $usemap = {
-#      8               => '',
-#      7               => '',
+sub usemap {{
        4               => 'WTI',
        1003    => 'WTI',
        16              => 'CU',
@@ -66,10 +63,6 @@ our $usemap = {
 
 };
 
-sub usemap {
-       my $f = shift || die;
-       $usemap->{$f};
-}
 
 sub search {
        my ( $self, $query ) = @_;
@@ -81,6 +74,7 @@ sub search {
 
 diag "get $url";
 
+       my $mech = $self->{mech} || die "no mech?";
        $mech->get( $url );
 
 diag "advanced search";
@@ -111,12 +105,16 @@ diag "got $hits results, get first one";
 diag "in MARC format";
 
        $mech->follow_link( url_regex => qr/format=001/ );
+
+       return $hits;
 }
 
 
 sub next_marc {
        my ($self,$format) = @_;
 
+       my $mech = $self->{mech} || die "no mech?";
+
 print $mech->content;
 
        if ( $mech->content =~ m{Zapis\s+(\d+)}s ) {
@@ -125,18 +123,31 @@ print $mech->content;
 
 diag "parse $nr";
 
+               my $marc = MARC::Record->new;
+
                my $html = $mech->content;
                my $hash;
-               $html =~ s|<tr>\s*<td class=td1 id=bold[^>]*>(.+?)</td>\s*<td class=td1>(.+?)</td>|$hash->{$1} = "$2";|ges;
+
+               sub field {
+                       my ( $f, $v ) = @_;
+                       $v =~ s/\Q&nbsp;\E/ /gs;
+warn "# $f\t$v\n";
+                       $hash->{$f} = $v;
+                       my ($i1,$i2) = (' ',' ');
+                       ($i1,$i2) = ($2,$3) if $f =~ s/^(...)(.)?(.)?/$1/;
+                       my @sf = split(/\|/, $v);
+                       shift @sf;
+                       @sf = map { s/^(\w)\s+//; { $1 => $_ } } @sf;
+diag "sf = ", dump(@sf);
+                       $marc->add_fields( $f, $i1, $i2, @sf ) if $f =~ m/^\d+$/;
+               }
+
+               $html =~ s|<tr>\s*<td class=td1 id=bold[^>]*>(.+?)</td>\s*<td class=td1>(.+?)</td>|field($1,$2)|ges;
                diag dump($hash);
 
                my $id = $hash->{SYS} || die "no SYS";
 
-die;
-
-               my $marc = MARC::Record->new;
 
-#              $marc->add_fields( $f, $i1, $i2, @{ $out->{$f} } );
 
                my $path = "marc/$id.$format";