added --parallel option to utilize multiple CPUs in machine

git-svn-id: svn+ssh://mjesec/home/dpavlin/svn/webpac2/trunk@606 07558da8-63fa-0310-ba24-9fe276d99e06

diff --git a/Makefile.PL b/Makefile.PL
index 11edce5..1f5349d 100644 (file)
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -32,6 +32,7 @@ WriteMakefile(
        'MARC::Record' => 2.0,
        'Data::Dump' => 0,
        'MARC::Lint' => 0,
+       'Proc::Queue' => 0,
     },
     dist                => { COMPRESS => 'gzip -9f', SUFFIX => 'gz', },
     clean               => { FILES => 'WebPAC-* pod2html Makefile tags' },

diff --git a/lib/WebPAC/Input.pm b/lib/WebPAC/Input.pm
index f4d9eb8..f748594 100644 (file)
--- a/lib/WebPAC/Input.pm
+++ b/lib/WebPAC/Input.pm
@@ -254,7 +254,7 @@ sub open {
 
                                ## FIXME remove this warning when we are sure that none of API is calling
                                ## this wrongly
-                               warn "filter called without field number" unless ($f_nr);
+                               #warn "filter called without field number" unless ($f_nr);
 
                                return $l unless ($rec_regex && $f_nr);
 

diff --git a/run.pl b/run.pl
index 971043d..4d8283f 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -22,6 +22,9 @@ use File::Slurp;
 use Data::Dump qw/dump/;
 use Storable qw/dclone/;
 
+use Proc::Queue size => 1;
+use POSIX ":sys_wait_h"; # imports WNOHANG
+
 =head1 NAME
 
 run.pl - start WebPAC indexing
@@ -81,6 +84,11 @@ messages with C<--no-marc-lint>.
 
 Force dump or input and marc record for debugging.
 
+=item --parallel 4
+
+Run databases in parallel (aproximatly same as number of processors in
+machine if you want to use full load)
+
 =back
 
 =cut
@@ -98,6 +106,8 @@ my ($marc_normalize, $marc_output);
 my $marc_lint = 1;
 my $marc_dump = 0;
 
+my $parallel = 0;
+
 GetOptions(
        "limit=i" => \$limit,
        "offset=i" => \$offset,
@@ -112,6 +122,7 @@ GetOptions(
        "marc-output=s" => \$marc_output,
        "marc-lint!" => \$marc_lint,
        "marc-dump!" => \$marc_dump,
+       "parallel=i" => \$parallel,
 );
 
 $config = LoadFile($config);
@@ -145,11 +156,25 @@ my $start_t = time();
 my @links;
 my $indexer;
 
+if ($parallel) {
+       $log->info("Using $parallel processes for speedup");
+       Proc::Queue::size($parallel);
+}
+
 while (my ($database, $db_config) = each %{ $config->{databases} }) {
 
        my ($only_database,$only_input) = split(m#/#, $only_filter) if ($only_filter);
        next if ($only_database && $database !~ m/$only_database/i);
 
+       if ($parallel) {
+               my $f=fork;
+               if(defined ($f) and $f==0) {
+                       $log->info("Created processes $$ for speedup");
+               } else {
+                       next;
+               }
+       }
+
        if ($use_indexer) {
                my $indexer_config = $config->{$use_indexer} || $log->logdie("can't find '$use_indexer' part in confguration");
                $indexer_config->{database} = $database;
@@ -390,9 +415,22 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
                }
        }
 
+       # end forked process
+       if ($parallel) {
+               $log->info("parallel process $$ finished");
+               exit(0);
+       }
+
+}
+
+if ($parallel) {
+       # wait all children to finish
+       sleep(1) while wait != -1;
+       $log->info("all parallel processes finished");
 }
 
 foreach my $link (@links) {
        $log->info("adding link $link->{from} -> $link->{to} [$link->{credit}]");
        $indexer->add_link( %{ $link } );
 }
+