r725@llin: dpavlin | 2006-06-29 15:48:38 +0200

 support arrays for normalize in config.yml [2.21]

git-svn-id: svn+ssh://mjesec/home/dpavlin/svn/webpac2/trunk@539 07558da8-63fa-0310-ba24-9fe276d99e06

diff --git a/TODO b/TODO
index 4171ec9..8a11923 100644 (file)
--- a/TODO
+++ b/TODO
@@ -17,7 +17,8 @@
 + add validator for input data [2.15]
 + add Excel input format [2.16]
 + remove WebPAC::Normalize::XML and promote WebPAC::Normalize::Set to WebPAC::Normalize [2.20]
-- support arrays for normalize/path and lookup
++ support arrays for normalize/path [2.21]
+- support arrays for lookup
 - add dBase input format
 - remove delimiters characters from index and query entered
 - delete unused files in database directories

diff --git a/lib/WebPAC.pm b/lib/WebPAC.pm
index 567b5c8..24b6fca 100644 (file)
--- a/lib/WebPAC.pm
+++ b/lib/WebPAC.pm
@@ -9,11 +9,11 @@ WebPAC - core module
 
 =head1 VERSION
 
-Version 2.20
+Version 2.21
 
 =cut
 
-our $VERSION = '2.20';
+our $VERSION = '2.21';
 
 =head1 SYNOPSIS
 

diff --git a/run.pl b/run.pl
index 825facd..3f06f35 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -220,56 +220,68 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
                        %{ $input },
                );
 
-               my $rules;
-               my $normalize_path = $input->{normalize}->{path};
+               my @norm_array = ref($input->{normalize}) eq 'ARRAY' ?
+                       @{ $input->{normalize} } : ( $input->{normalize} );
 
-               $log->logdie("Found '$normalize_path' as normalization file which isn't supported any more!") unless ( $normalize_path =~ m!\.pl$!i );
+               foreach my $normalize (@norm_array) {
 
-               my $rules = read_file( $normalize_path ) or die "can't open $normalize_path: $!";
+                       my $rules;
+                       my $normalize_path = $normalize->{path} || $log->logdie("can't find normalize path in config");
 
-               foreach my $pos ( 0 ... $input_db->size ) {
+                       $log->logdie("Found '$normalize_path' as normalization file which isn't supported any more!") unless ( $normalize_path =~ m!\.pl$!i );
 
-                       my $row = $input_db->fetch || next;
+                       my $rules = read_file( $normalize_path ) or die "can't open $normalize_path: $!";
 
-                       my $mfn = $row->{'000'}->[0];
+                       $log->info("Using $normalize_path for normalization...");
+
+                       # reset position in database
+                       $input_db->seek(1);
+
+                       foreach my $pos ( 0 ... $input_db->size ) {
+
+                               my $row = $input_db->fetch || next;
+
+                               my $mfn = $row->{'000'}->[0];
+
+                               if (! $mfn || $mfn !~ m#^\d+$#) {
+                                       $log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos");
+                                       $mfn = $pos;
+                                       push @{ $row->{'000'} }, $pos;
+                               }
 
-                       if (! $mfn || $mfn !~ m#^\d+$#) {
-                               $log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos");
-                               $mfn = $pos;
-                               push @{ $row->{'000'} }, $pos;
-                       }
 
+                               if ($validate) {
+                                       my @errors = $validate->validate_errors( $row );
+                                       $log->error( "MFN $mfn validation errors:\n", join("\n", @errors) ) if (@errors);
+                               }
 
-                       if ($validate) {
-                               my @errors = $validate->validate_errors( $row );
-                               $log->error( "MFN $mfn validation errors:\n", join("\n", @errors) ) if (@errors);
+                                       
+                               my $ds = WebPAC::Normalize::data_structure(
+                                       row => $row,
+                                       rules => $rules,
+                                       lookup => $lookup ? $lookup->lookup_hash : undef,
+                               );
+
+                               $db->save_ds(
+                                       id => $mfn,
+                                       ds => $ds,
+                                       prefix => $input->{name},
+                               ) if ($ds && !$stats);
+
+                               $indexer->add(
+                                       id => $input->{name} . "/" . $mfn,
+                                       ds => $ds,
+                                       type => $config->{$use_indexer}->{type},
+                               ) if ($indexer);
+
+                               $total_rows++;
                        }
 
-                               
-                       my $ds = WebPAC::Normalize::data_structure(
-                               row => $row,
-                               rules => $rules,
-                               lookup => $lookup ? $lookup->lookup_hash : undef,
-                       );
-
-                       $db->save_ds(
-                               id => $mfn,
-                               ds => $ds,
-                               prefix => $input->{name},
-                       ) if ($ds && !$stats);
-
-                       $indexer->add(
-                               id => $input->{name} . "/" . $mfn,
-                               ds => $ds,
-                               type => $config->{$use_indexer}->{type},
-                       ) if ($indexer);
-
-                       $total_rows++;
-               }
+                       $log->info("statistics of fields usage:\n", $input_db->stats) if ($stats);
 
-               $log->info("statistics of fields usage:\n", $input_db->stats) if ($stats);
+               };
 
-       };
+       }
 
        eval { $indexer->finish } if ($indexer && $indexer->can('finish'));