added --parallel option to utilize multiple CPUs in machine

[webpac2] / run.pl

diff --git a/run.pl b/run.pl
index 971043d..4d8283f 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -22,6 +22,9 @@ use File::Slurp;
 use Data::Dump qw/dump/;
 use Storable qw/dclone/;
 
+use Proc::Queue size => 1;
+use POSIX ":sys_wait_h"; # imports WNOHANG
+
 =head1 NAME
 
 run.pl - start WebPAC indexing
@@ -81,6 +84,11 @@ messages with C<--no-marc-lint>.
 
 Force dump or input and marc record for debugging.
 
+=item --parallel 4
+
+Run databases in parallel (aproximatly same as number of processors in
+machine if you want to use full load)
+
 =back
 
 =cut
@@ -98,6 +106,8 @@ my ($marc_normalize, $marc_output);
 my $marc_lint = 1;
 my $marc_dump = 0;
 
+my $parallel = 0;
+
 GetOptions(
        "limit=i" => \$limit,
        "offset=i" => \$offset,
@@ -112,6 +122,7 @@ GetOptions(
        "marc-output=s" => \$marc_output,
        "marc-lint!" => \$marc_lint,
        "marc-dump!" => \$marc_dump,
+       "parallel=i" => \$parallel,
 );
 
 $config = LoadFile($config);
@@ -145,11 +156,25 @@ my $start_t = time();
 my @links;
 my $indexer;
 
+if ($parallel) {
+       $log->info("Using $parallel processes for speedup");
+       Proc::Queue::size($parallel);
+}
+
 while (my ($database, $db_config) = each %{ $config->{databases} }) {
 
        my ($only_database,$only_input) = split(m#/#, $only_filter) if ($only_filter);
        next if ($only_database && $database !~ m/$only_database/i);
 
+       if ($parallel) {
+               my $f=fork;
+               if(defined ($f) and $f==0) {
+                       $log->info("Created processes $$ for speedup");
+               } else {
+                       next;
+               }
+       }
+
        if ($use_indexer) {
                my $indexer_config = $config->{$use_indexer} || $log->logdie("can't find '$use_indexer' part in confguration");
                $indexer_config->{database} = $database;
@@ -390,9 +415,22 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
                }
        }
 
+       # end forked process
+       if ($parallel) {
+               $log->info("parallel process $$ finished");
+               exit(0);
+       }
+
+}
+
+if ($parallel) {
+       # wait all children to finish
+       sleep(1) while wait != -1;
+       $log->info("all parallel processes finished");
 }
 
 foreach my $link (@links) {
        $log->info("adding link $link->{from} -> $link->{to} [$link->{credit}]");
        $indexer->add_link( %{ $link } );
 }
+