r772@llin: dpavlin | 2006-07-02 22:14:37 +0200

[webpac2] / run.pl

diff --git a/run.pl b/run.pl
index d69d16d..f374abf 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -4,7 +4,6 @@ use strict;
 
 use Cwd qw/abs_path/;
 use File::Temp qw/tempdir/;
-use Data::Dumper;
 use lib './lib';
 
 use WebPAC::Common 0.02;
@@ -20,6 +19,8 @@ use File::Path;
 use Time::HiRes qw/time/;
 use File::Slurp;
 use MARC::Record 2.0;  # need 2.0 for utf-8 encoding see marcpm.sf.net
+use MARC::Lint;
+use Data::Dump qw/dump/;
 
 =head1 NAME
 
@@ -63,6 +64,23 @@ usage for each input
 
 turn on extra validation of imput records, see L<WebPAC::Validation>
 
+=item --marc-normalize conf/normalize/mapping.pl
+
+This option specifies normalisation file for MARC creation
+
+=item --marc-output out/marc/test.marc
+
+Optional path to output file
+
+=item --marc-lint
+
+By default turned on if C<--marc-normalize> is used. You can disable lint
+messages with C<--no-marc-lint>.
+
+=item --marc-dump
+
+Force dump or input and marc record for debugging.
+
 =back
 
 =cut
@@ -76,6 +94,9 @@ my $debug = 0;
 my $only_filter;
 my $stats = 0;
 my $validate_path;
+my ($marc_normalize, $marc_output);
+my $marc_lint = 1;
+my $marc_dump = 0;
 
 GetOptions(
        "limit=i" => \$limit,
@@ -84,14 +105,18 @@ GetOptions(
        "one=s" => \$only_filter,
        "only=s" => \$only_filter,
        "config" => \$config,
-       "debug" => \$debug,
+       "debug+" => \$debug,
        "stats" => \$stats,
        "validate=s" => \$validate_path,
+       "marc-normalize=s" => \$marc_normalize,
+       "marc-output=s" => \$marc_output,
+       "marc-lint!" => \$marc_lint,
+       "marc-dump!" => \$marc_dump,
 );
 
 $config = LoadFile($config);
 
-print "config = ",Dumper($config) if ($debug);
+print "config = ",dump($config) if ($debug);
 
 die "no databases in config file!\n" unless ($config->{databases});
 
@@ -111,12 +136,17 @@ if ($stats) {
        $log->info("using $use_indexer indexing engine...");
 }
 
+# disable indexing when creating marc
+$use_indexer = undef if ($marc_normalize);
+
 my $total_rows = 0;
 my $start_t = time();
 
 my @links;
 my $indexer;
 
+my $lint = new MARC::Lint if ($marc_lint);
+
 while (my ($database, $db_config) = each %{ $config->{databases} }) {
 
        my ($only_database,$only_input) = split(m#/#, $only_filter) if ($only_filter);
@@ -224,6 +254,13 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
                my @norm_array = ref($input->{normalize}) eq 'ARRAY' ?
                        @{ $input->{normalize} } : ( $input->{normalize} );
 
+               if ($marc_normalize) {
+                       @norm_array = ( {
+                               path => $marc_normalize,
+                               output => $marc_output || 'out/marc/' . $database . '-' . $input->{name} . '.marc',
+                       } );
+               }
+
                foreach my $normalize (@norm_array) {
 
                        my $normalize_path = $normalize->{path} || $log->logdie("can't find normalize path in config");
@@ -235,11 +272,11 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
                        $log->info("Using $normalize_path for normalization...");
 
                        my $marc_fh;
-                       if (my $path = $normalize->{marc21}) {
+                       if (my $path = $normalize->{output}) {
                                open($marc_fh, '>', $path) ||
                                        $log->logdie("can't open MARC output $path: $!");
 
-                               $log->info("Creating MARC export file $path\n");
+                               $log->info("Creating MARC export file $path", $marc_lint ? ' (with lint)' : '', "\n");
                        }
 
                        # reset position in database
@@ -263,7 +300,6 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
                                        $log->error( "MFN $mfn validation errors:\n", join("\n", @errors) ) if (@errors);
                                }
 
-                                       
                                my $ds = WebPAC::Normalize::data_structure(
                                        row => $row,
                                        rules => $rules,
@@ -286,8 +322,44 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
                                if ($marc_fh) {
                                        my $marc = new MARC::Record;
                                        $marc->encoding( 'utf-8' );
-                                       $marc->add_fields( WebPAC::Normalize::_get_marc21_fields() );
-                                       print $marc_fh $marc->as_usmarc;
+                                       my @marc_fields = WebPAC::Normalize::_get_marc_fields();
+                                       if (! @marc_fields) {
+                                               $log->warn("MARC record $mfn is empty, skipping");
+                                       } else {
+                                               $marc->add_fields( @marc_fields );
+
+                                               # tweak leader
+                                               if (my $new_l = WebPAC::Normalize::marc_leader()) {
+
+                                                       my $leader = $marc->leader;
+
+                                                       foreach my $o ( keys %$new_l ) {
+                                                               my $insert = $new_l->{$o};
+                                                               $leader = substr($leader, 0, $o) .
+                                                                       $insert . substr($leader, $o+length($insert));
+                                                       }
+                                                       $marc->leader( $leader );
+                                               }
+
+                                               if ($marc_lint) {
+                                                       $lint->check_record( $marc );
+                                                       my $err = join( "\n", $lint->warnings );
+                                                       $log->error("MARC lint detected warning on MFN $mfn\n",
+                                                               "Original imput row: ",dump($row), "\n",
+                                                               "Normalized MARC row: ",dump(@marc_fields), "\n",
+                                                               "MARC lint warnings: ",$err,"\n"
+                                                       ) if ($err);
+                                               }
+
+                                               if ($marc_dump) {
+                                                       $log->info("MARC record on MFN $mfn\tleader:", $marc->leader(), "\n",
+                                                               "Original imput row: ",dump($row), "\n",
+                                                               "Normalized MARC row: ",dump(@marc_fields), "\n",
+                                                       );
+                                               }
+
+                                               print $marc_fh $marc->as_usmarc;
+                                       }
                                }
 
                                $total_rows++;
@@ -314,7 +386,7 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
        #
        # add Hyper Estraier links to other databases
        #
-       if (ref($db_config->{links}) eq 'ARRAY') {
+       if (ref($db_config->{links}) eq 'ARRAY' && $use_indexer) {
                foreach my $link (@{ $db_config->{links} }) {
                        if ($use_indexer eq 'hyperestraier') {
                                $log->info("saving link $database -> $link->{to} [$link->{credit}]");