r1850@llin: dpavlin | 2009-04-26 02:07:56 +0200

[webpac2] / run.pl

diff --git a/run.pl b/run.pl
index cd392f0..8b3d666 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -22,8 +22,8 @@ use File::Slurp;
 use Data::Dump qw/dump/;
 use Storable qw/dclone/;
 use Pod::Usage qw/pod2usage/;
+use LWP::Simple;
 
-use Proc::Queue size => 1;
 use POSIX ":sys_wait_h"; # imports WNOHANG
 
 =head1 NAME
@@ -103,6 +103,10 @@ Create just links
 
 Create merged index of databases which have links
 
+=item --mirror http://www.example.com
+
+Tries to download input path files from mirror URI
+
 =back
 
 =cut
@@ -123,6 +127,7 @@ my $marc_dump = 0;
 my $parallel = 0;
 my $only_links = 0;
 my $merge = 0;
+my $mirror;
 my $help;
 
 my $log = _new WebPAC::Common()->_get_logger();
@@ -144,6 +149,7 @@ GetOptions(
        "parallel=i" => \$parallel,
        "only-links!" => \$only_links,
        "merge" => \$merge,
+       "mirror=s" => \$mirror,
        "help" => \$help,
 );
 
@@ -207,6 +213,8 @@ my $start_t = time();
 my @links;
 
 if ($parallel) {
+       eval 'use Proc::Queue size => 1;';
+       die $@ if $@;
        $log->info("Using $parallel processes for speedup");
        Proc::Queue::size($parallel);
 }
@@ -349,7 +357,7 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                $log->info("database $database doesn't have inputs defined");
        } );
 
-       if ( -e 'out/debug' ) { # fixme flag?
+       if ( -e 'out/debug' ) { # FIXME flag?
                my $out;
                foreach my $i ( @inputs ) {
                        warn dump( $i );
@@ -370,6 +378,11 @@ foreach my $database ( sort keys %{ $config->databases } ) {
 
                my $input_name = $input->{name} || $log->logdie("input without a name isn't valid: ",dump($input));
 
+               if ( $input->{skip} ) {
+                       $log->info("skip $input_name");
+                       next;
+               }
+
                next if ($only_input && ($input_name !~ m#$only_input#i && $input->{type} !~ m#$only_input#i));
 
                my $type = lc($input->{type});
@@ -397,9 +410,16 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                        delete($input->{modify_file});
                }
 
+               if ( $mirror ) {
+                       my $path = $input->{path} || die "no input path in ",dump( $input );
+                       $log->info( "mirror ", $path, " ", -s $path, " bytes" );
+
+                       $log->warn( "$path not modified" )
+                               if mirror( "$mirror/$path", $path ) == RC_NOT_MODIFIED;
+               }
+
                my $input_db = new WebPAC::Input(
                        module => $input_module,
-                       encoding => $config->webpac('webpac_encoding'),
                        limit => $limit || $input->{limit},
                        offset => $offset,
                        recode => $input->{recode},
@@ -444,7 +464,7 @@ foreach my $database ( sort keys %{ $config->databases } ) {
 
                my $maxmfn = $input_db->open(
                        path => $input->{path},
-                       code_page => $input->{encoding},        # database encoding
+                       input_encoding => $input->{encoding},   # database encoding
                        lookup_coderef => $lookup_coderef,
                        lookup => $lookup_jar,
                        %{ $input },
@@ -668,7 +688,7 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                close($report_fh) if ($report_fh);
        }
 
-       eval { $indexer->finish } if ($indexer && $indexer->can('finish'));
+       $indexer->finish if $indexer && $indexer->can('finish');
 
        foreach my $out ( @output_modules ) {
                $out->finish if $out->can('finish');