r1325@llin: dpavlin | 2007-09-03 17:28:32 +0200

[webpac2] / run.pl

diff --git a/run.pl b/run.pl
index 9e69a83..26c6b9a 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -9,7 +9,7 @@ use lib './lib';
 use WebPAC::Common 0.02;
 use WebPAC::Parser 0.08;
 use WebPAC::Input 0.16;
-use WebPAC::Store 0.14;
+use WebPAC::Store 0.15;
 use WebPAC::Normalize 0.22;
 use WebPAC::Output::TT;
 use WebPAC::Validate 0.11;
@@ -185,7 +185,7 @@ $stats ||= $validate;
 if ($stats) {
        $log->debug("disabled indexing for stats collection");
        $use_indexer = undef;
-} else {
+} elsif ( $use_indexer ) {
        $log->info("using $use_indexer indexing engine...");
 }
 
@@ -244,19 +244,19 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                if ($use_indexer eq 'hyperestraier') {
 
                        # open Hyper Estraier database
-                       use WebPAC::Output::Estraier '0.10';
+                       require WebPAC::Output::Estraier;
                        $indexer = new WebPAC::Output::Estraier( %{ $indexer_config } );
                
                } elsif ($use_indexer eq 'hyperestraier-native') {
 
                        # open Hyper Estraier database
-                       use WebPAC::Output::EstraierNative;
+                       require WebPAC::Output::EstraierNative;
                        $indexer = new WebPAC::Output::EstraierNative( %{ $indexer_config } );
 
                } elsif ($use_indexer eq 'kinosearch') {
 
                        # open KinoSearch
-                       use WebPAC::Output::KinoSearch;
+                       require WebPAC::Output::KinoSearch;
                        $indexer_config->{clean} = 1 unless (-e $indexer_config->{index_path});
                        $indexer = new WebPAC::Output::KinoSearch( %{ $indexer_config } );
 
@@ -264,7 +264,7 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                        $log->logdie("unknown use_indexer: $use_indexer");
                }
 
-               $log->logide("can't continue without valid indexer") unless ($indexer);
+               $log->logdie("can't continue without valid indexer") unless ($indexer);
        }
 
 
@@ -298,23 +298,24 @@ foreach my $database ( sort keys %{ $config->databases } ) {
        #
        # now WebPAC::Store
        #
-       my $abs_path = abs_path($0);
-       $abs_path =~ s#/[^/]*$#/#;
-
-       my $db_path = $config->webpac('db_path');
+       my $store = new WebPAC::Store({
+               debug => $debug,
+       });
 
-       if ($clean) {
-               $log->info("creating new database '$database' in $db_path");
-               rmtree( $db_path ) || $log->warn("can't remove $db_path: $!");
-       } else {
-               $log->info("working on database '$database' in $db_path");
+       #
+       # prepare output
+       #
+       my @outputs;
+       if (defined( $db_config->{output} )) {
+               my $module = $db_config->{output}->{module} || $log->logdie("need module in output section of $database");
+               $module = 'WebPAC::Output::' . $module unless $module =~ m/::/;
+               $log->debug("loading output module $module");
+               eval "require $module";
+               my $out = new $module->new( $db_config->{output} );
+               $out->init;
+               push @outputs, $out;
        }
 
-       my $store = new WebPAC::Store(
-               path => $db_path,
-               debug => $debug,
-       );
-
 
        #
        # now, iterate through input formats
@@ -362,6 +363,7 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                        stats => $stats,
                        modify_records => $input->{modify_records},
                        modify_file => $input->{modify_file},
+                       input_config => $input,
                );
                $log->logdie("can't create input using $input_module") unless ($input);
 
@@ -448,8 +450,8 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                        $log->info("Generating report file $path");
 
                        if ( $validate ) {
-                               $validate->read_validate_file( $validate->fill_in( $validate_path, database => $database, input => $input_name ) );
-                               $validate->read_validate_delimiters_file( $validate->fill_in( $validate_delimiters_path, database => $database, input => $input_name ) );
+                               $validate->read_validate_file( $validate->fill_in( $validate_path, database => $database, input => $input_name ) ) if ( $validate_path );
+                               $validate->read_validate_delimiters_file( $validate->fill_in( $validate_delimiters_path, database => $database, input => $input_name ) ) if ( $validate_delimiters_path );
                        }
                }
 
@@ -506,7 +508,7 @@ foreach my $database ( sort keys %{ $config->databases } ) {
 
                        my $mfn = $row->{'000'}->[0];
 
-                       if (! $mfn || $mfn !~ m#^\d+$#) {
+                       if (! $mfn || $mfn !~ m{^\d+$}) {
                                $log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos");
                                $mfn = $pos;
                                push @{ $row->{'000'} }, $pos;
@@ -568,6 +570,11 @@ foreach my $database ( sort keys %{ $config->databases } ) {
 
                                $log->info("Created $i instances of MFN $mfn\n") if ($i > 1);
                        }
+
+                       foreach my $out ( @outputs ) {
+                               $out->add( $mfn, $ds ) if $out->can('add');
+                       }
+
                }
 
                if ($validate) {
@@ -598,6 +605,10 @@ foreach my $database ( sort keys %{ $config->databases } ) {
 
        eval { $indexer->finish } if ($indexer && $indexer->can('finish'));
 
+       foreach my $out ( @outputs ) {
+               $out->finish if $out->can('finish');
+       }
+
        my $dt = time() - $start_t;
        $log->info("$total_rows records ", $indexer ? "indexed " : "",
                sprintf("in %.2f sec [%.2f rec/sec]",
@@ -621,7 +632,7 @@ if ($parallel) {
 }
 
 # save new delimiters if needed
-$validate->save_delimiters_templates if ( $validate );
+$validate->save_delimiters_templates if ( $validate_delimiters_path );
 
 #
 # handle links or merge after indexing