r1285@llin: dpavlin | 2007-06-21 14:53:43 +0200

[webpac2] / run.pl

diff --git a/run.pl b/run.pl
index 72ccca1..036b1bb 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -12,7 +12,7 @@ use WebPAC::Input 0.16;
 use WebPAC::Store 0.14;
 use WebPAC::Normalize 0.22;
 use WebPAC::Output::TT;
-use WebPAC::Validate 0.06;
+use WebPAC::Validate 0.11;
 use WebPAC::Output::MARC;
 use WebPAC::Config;
 use Getopt::Long;
@@ -68,6 +68,14 @@ and subfield usage for each input
 
 turn on extra validation of imput records, see L<WebPAC::Validation>
 
+You can use special variables C<$database> and $C<$input> in this parametar
+like C<--validate 'conf/validate/$database-$input'> to construct filename
+
+=item --validate-delimiters path/to/validate_delimiters_file
+
+this option is used with C<--validate> to turn on extra validation of
+delimiters. If file is non existant, it will be created on first run.
+
 =item --marc-generate
 
 Generate MARC file. This will automatically be on if file contains C<marc*> directives.
@@ -108,6 +116,7 @@ my $debug = 0;
 my $only_filter;
 my $stats = 0;
 my $validate_path;
+my $validate_delimiters_path;
 my $marc_generate = 1;
 my $marc_lint = 1;
 my $marc_dump = 0;
@@ -128,6 +137,7 @@ GetOptions(
        "debug+" => \$debug,
        "stats" => \$stats,
        "validate=s" => \$validate_path,
+       "validate-delimiters=s" => \$validate_delimiters_path,
        "marc-generate!" => \$marc_generate,
        "marc-lint!" => \$marc_lint,
        "marc-dump!" => \$marc_dump,
@@ -137,6 +147,8 @@ GetOptions(
        "help" => \$help,
 );
 
+$marc_generate = 0 if ( $validate_delimiters_path );
+
 pod2usage(-verbose => 2) if ($help);
 
 my $config = new WebPAC::Config( path => $config_path );
@@ -163,19 +175,17 @@ if ($merge) {
        $log->info("created merge batch file $estcmd_path");
 }
 
-
 my $validate;
 $validate = new WebPAC::Validate(
-       path => $validate_path,
-) if ($validate_path);
-
+       delimiters => $config->webpac('delimiters'),
+) if ($validate_path || $validate_delimiters_path);
 
 my $use_indexer = $config->use_indexer;
 $stats ||= $validate;
 if ($stats) {
        $log->debug("disabled indexing for stats collection");
        $use_indexer = undef;
-} else {
+} elsif ( $use_indexer ) {
        $log->info("using $use_indexer indexing engine...");
 }
 
@@ -202,7 +212,8 @@ sub create_ds_config {
        return $c;
 }
 
-while (my ($database, $db_config) = each %{ $config->databases }) {
+foreach my $database ( sort keys %{ $config->databases } ) {
+       my $db_config = $config->databases->{$database};
 
        my ($only_database,$only_input) = split(m#/#, $only_filter) if ($only_filter);
        next if ($only_database && $database !~ m/$only_database/i);
@@ -233,19 +244,19 @@ while (my ($database, $db_config) = each %{ $config->databases }) {
                if ($use_indexer eq 'hyperestraier') {
 
                        # open Hyper Estraier database
-                       use WebPAC::Output::Estraier '0.10';
+                       require WebPAC::Output::Estraier;
                        $indexer = new WebPAC::Output::Estraier( %{ $indexer_config } );
                
                } elsif ($use_indexer eq 'hyperestraier-native') {
 
                        # open Hyper Estraier database
-                       use WebPAC::Output::EstraierNative;
+                       require WebPAC::Output::EstraierNative;
                        $indexer = new WebPAC::Output::EstraierNative( %{ $indexer_config } );
 
                } elsif ($use_indexer eq 'kinosearch') {
 
                        # open KinoSearch
-                       use WebPAC::Output::KinoSearch;
+                       require WebPAC::Output::KinoSearch;
                        $indexer_config->{clean} = 1 unless (-e $indexer_config->{index_path});
                        $indexer = new WebPAC::Output::KinoSearch( %{ $indexer_config } );
 
@@ -253,7 +264,7 @@ while (my ($database, $db_config) = each %{ $config->databases }) {
                        $log->logdie("unknown use_indexer: $use_indexer");
                }
 
-               $log->logide("can't continue without valid indexer") unless ($indexer);
+               $log->logdie("can't continue without valid indexer") unless ($indexer);
        }
 
 
@@ -435,6 +446,11 @@ while (my ($database, $db_config) = each %{ $config->databases }) {
                        print $report_fh "Report for database '$database' input '$input_name' records ",
                                $offset || 1, "-", $limit || $input->{limit} || $maxmfn, "\n\n";
                        $log->info("Generating report file $path");
+
+                       if ( $validate ) {
+                               $validate->read_validate_file( $validate->fill_in( $validate_path, database => $database, input => $input_name ) ) if ( $validate_path );
+                               $validate->read_validate_delimiters_file( $validate->fill_in( $validate_delimiters_path, database => $database, input => $input_name ) ) if ( $validate_delimiters_path );
+                       }
                }
 
                my $marc;
@@ -544,7 +560,7 @@ while (my ($database, $db_config) = each %{ $config->databases }) {
                                        $marc->add(
                                                id => $mfn . ( $i ? "/$i" : '' ),
                                                fields => $fields,
-                                               leader => WebPAC::Normalize::marc_leader(),
+                                               leader => WebPAC::Normalize::_get_marc_leader(),
                                                row => $row,
                                        );
                                        $i++;
@@ -560,6 +576,11 @@ while (my ($database, $db_config) = each %{ $config->databases }) {
                                $log->info("validation errors:\n$errors\n" );
                                print $report_fh "$errors\n" if ($report_fh);
                        }
+
+                       print $report_fh "\nAll possible subfields/delimiter templates:\n", $validate->delimiters_templates( report => 1, current_input => 1 ), "\n\n";
+
+                       # must be last thing that touches $validate for this input
+                       $validate->reset;
                }
 
                if ($stats) {
@@ -572,8 +593,7 @@ while (my ($database, $db_config) = each %{ $config->databases }) {
                $marc->finish if ($marc);
 
                # close report
-               close($report_fh) if ($report_fh)
-
+               close($report_fh) if ($report_fh);
        }
 
        eval { $indexer->finish } if ($indexer && $indexer->can('finish'));
@@ -600,6 +620,9 @@ if ($parallel) {
        $log->info("all parallel processes finished");
 }
 
+# save new delimiters if needed
+$validate->save_delimiters_templates if ( $validate_delimiters_path );
+
 #
 # handle links or merge after indexing
 #