r1813@llin: dpavlin | 2009-04-25 11:28:36 +0200

[webpac2] / run.pl

diff --git a/run.pl b/run.pl
index c3d3b77..01d3e77 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -23,7 +23,6 @@ use Data::Dump qw/dump/;
 use Storable qw/dclone/;
 use Pod::Usage qw/pod2usage/;
 
-use Proc::Queue size => 1;
 use POSIX ":sys_wait_h"; # imports WNOHANG
 
 =head1 NAME
@@ -207,6 +206,8 @@ my $start_t = time();
 my @links;
 
 if ($parallel) {
+       eval 'use Proc::Queue size => 1;';
+       die $@ if $@;
        $log->info("Using $parallel processes for speedup");
        Proc::Queue::size($parallel);
 }
@@ -349,6 +350,23 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                $log->info("database $database doesn't have inputs defined");
        } );
 
+       if ( -e 'out/debug' ) { # FIXME flag?
+               my $out;
+               foreach my $i ( @inputs ) {
+                       warn dump( $i );
+                       next unless defined $i->{normalize};
+                       warn dump( $i->{normalize} );
+                       foreach my $normalize ( @{ $i->{normalize} } ) {
+                               my $path = $normalize->{path};
+                               $out .= qq/\n##\n## $path\n##\n\n/;
+                               $out .= read_file( $path );
+                       }
+               }
+               my $all = "out/debug/all-normalize.pl";
+               write_file( $all, $out );
+               warn "### all normalize for this input saved to: $all";
+       };
+
        foreach my $input (@inputs) {
 
                my $input_name = $input->{name} || $log->logdie("input without a name isn't valid: ",dump($input));
@@ -382,7 +400,6 @@ foreach my $database ( sort keys %{ $config->databases } ) {
 
                my $input_db = new WebPAC::Input(
                        module => $input_module,
-                       encoding => $config->webpac('webpac_encoding'),
                        limit => $limit || $input->{limit},
                        offset => $offset,
                        recode => $input->{recode},
@@ -427,7 +444,7 @@ foreach my $database ( sort keys %{ $config->databases } ) {
 
                my $maxmfn = $input_db->open(
                        path => $input->{path},
-                       code_page => $input->{encoding},        # database encoding
+                       input_encoding => $input->{encoding},   # database encoding
                        lookup_coderef => $lookup_coderef,
                        lookup => $lookup_jar,
                        %{ $input },
@@ -651,7 +668,7 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                close($report_fh) if ($report_fh);
        }
 
-       eval { $indexer->finish } if ($indexer && $indexer->can('finish'));
+       $indexer->finish if $indexer && $indexer->can('finish');
 
        foreach my $out ( @output_modules ) {
                $out->finish if $out->can('finish');