added no warnings 'redefine'; so we can define subs in normalization perl

[webpac2] / run.pl

diff --git a/run.pl b/run.pl
index 03de8b3..2dc7eb7 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -133,7 +133,7 @@ GetOptions(
        "clean" => \$clean,
        "one=s" => \$only_filter,
        "only=s" => \$only_filter,
-       "config" => \$config_path,
+       "config=s" => \$config_path,
        "debug+" => \$debug,
        "stats" => \$stats,
        "validate=s" => \$validate_path,
@@ -191,7 +191,13 @@ if ($stats) {
 
 # parse normalize files and create source files for lookup and normalization
 
-my $parser = new WebPAC::Parser( config => $config );
+my ($only_database,$only_input) = split(m#/#, $only_filter) if $only_filter;
+
+my $parser = new WebPAC::Parser(
+       config => $config,
+       only_database => $only_database,
+       only_input => $only_input,
+);
 
 my $total_rows = 0;
 my $start_t = time();
@@ -215,7 +221,6 @@ sub create_ds_config {
 foreach my $database ( sort keys %{ $config->databases } ) {
        my $db_config = $config->databases->{$database};
 
-       my ($only_database,$only_input) = split(m#/#, $only_filter) if ($only_filter);
        next if ($only_database && $database !~ m/$only_database/i);
 
        if ($parallel) {
@@ -299,23 +304,11 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                debug => $debug,
        });
 
-       sub iterate_over {
-               my ( $what, $error ) = @_;
-               my @result;
-               if ( ref( $what ) eq 'ARRAY' ) {
-                       @result = @{ $what };
-               } elsif ($db_config->{input}) {
-                       @result =  ( $what );
-               } else {
-                       $error->() if ref($error) eq 'CODE';
-               }
-               return @result;
-       }
 
        #
        # prepare output
        #
-       my @outputs = iterate_over( $db_config->{output}, sub {
+       my @outputs = force_array( $db_config->{output}, sub {
                $log->error("Database $database doesn't have any outputs defined. Do you want to remove it from configuration?" );
        } );
 
@@ -323,14 +316,15 @@ foreach my $database ( sort keys %{ $config->databases } ) {
 
        foreach my $output ( @outputs ) {
 
-warn '## output = ',dump( $output );
+#warn '## output = ',dump( $output );
 
                my $module = $output->{module} || $log->logdie("need module in output section of $database");
                $module = 'WebPAC::Output::' . $module unless $module =~ m/::/;
        
                $log->debug("loading output module $module");
                eval "require $module";
-       
+
+               # add database to arugemnts for output filter
                $output->{database} = $database;
 
                $log->debug("calling $module->new(",dump( $output ),")");
@@ -346,7 +340,7 @@ warn '## output = ',dump( $output );
        #
 
 
-       my @inputs = iterate_over( $db_config->{input}, sub {
+       my @inputs = force_array( $db_config->{input}, sub {
                $log->info("database $database doesn't have inputs defined");
        } );
 
@@ -358,9 +352,16 @@ warn '## output = ',dump( $output );
 
                my $type = lc($input->{type});
 
-               die "I know only how to handle input types ", join(",", $config->webpac('inputs') ), " not '$type'!\n" unless (grep(/$type/, $config->webpac('inputs')));
+               # FIXME check if input module exists
+               my $input_module = $input->{module};
 
-               my $input_module = $config->webpac('inputs')->{$type};
+               if ( ! $input_module ) {
+                       if ( grep(/$type/, $config->webpac('inputs')) ) {
+                               $input_module = $config->webpac('inputs')->{$type};
+                       } else {
+                               $log->logdie("I know only how to handle input types ", join(",", $config->webpac('inputs') ), " not '$type'!" );
+                       }
+               }
 
                my @lookups = $parser->have_lookup_create($database, $input);
 
@@ -520,6 +521,16 @@ warn '## output = ',dump( $output );
                }
 
 
+               # setup input name for all output filters
+               foreach my $out ( @output_modules ) {
+                       if ( $out->can('input') ) {
+                               $out->input( $input_name );
+                       } else {
+                               $log->warn("output filter ",ref($out)," doesn't support input name");
+                       }
+               }
+
+
                foreach my $pos ( 0 ... $input_db->size ) {
 
                        my $row = $input_db->fetch || next;
@@ -552,6 +563,7 @@ warn '## output = ',dump( $output );
                                marc_encoding => 'utf-8',
                                load_row_coderef => sub {
                                        my ($database,$input,$mfn) = @_;
+#warn "### load_row($database,$input,$mfn) from data_structure\n";
                                        return $store->load_row(
                                                database => $database,
                                                input => $input,
@@ -560,20 +572,30 @@ warn '## output = ',dump( $output );
                                },
                        );
 
-                       $log->debug("ds = ", sub { dump($ds) }) if ($ds);
+                       $log->debug("ds = ", sub { dump($ds) });
 
-                       $store->save_ds(
-                               database => $database,
-                               input => $input_name,
-                               id => $mfn,
-                               ds => $ds,
-                       ) if ($ds && !$stats);
+                       if ( $ds ) {
 
-                       $indexer->add(
-                               id => "${input_name}/${mfn}",
-                               ds => $ds,
-                               type => $config->get($indexer_config)->{type},
-                       ) if ($indexer && $ds);
+                               $store->save_ds(
+                                       database => $database,
+                                       input => $input_name,
+                                       id => $mfn,
+                                       ds => $ds,
+                               ) if !$stats;
+
+                               $indexer->add(
+                                       id => "${input_name}/${mfn}",
+                                       ds => $ds,
+                                       type => $config->get($indexer_config)->{type},
+                               ) if $indexer;
+
+                               foreach my $out ( @output_modules ) {
+                                       $out->add( $mfn, $ds ) if $out->can('add');
+                               }
+
+                       } else {
+                               $log->warn("record $pos didn't produce any output after normalization rules!") unless $marc;
+                       }
 
                        if ($marc) {
                                my $i = 0;
@@ -591,10 +613,6 @@ warn '## output = ',dump( $output );
                                $log->info("Created $i instances of MFN $mfn\n") if ($i > 1);
                        }
 
-                       foreach my $out ( @output_modules ) {
-                               $out->add( $mfn, $ds ) if $out->can('add');
-                       }
-
                }
 
                if ($validate) {