r322@athlon: dpavlin | 2005-12-19 22:27:06 +0100

 make run.pl moderatly chatty (along with other modules), added command line options
 (try perldoc run.pl) new target index (to reindex all) and run (to index
 first 100 records of each database)

git-svn-id: svn+ssh://mjesec/home/dpavlin/svn/webpac2/trunk@301 07558da8-63fa-0310-ba24-9fe276d99e06

diff --git a/MANIFEST b/MANIFEST
index 7c9470e..469227b 100644 (file)
--- a/MANIFEST
+++ b/MANIFEST
@@ -31,7 +31,6 @@ t/winisis/BIBL.N02
 t/winisis/BIBL.XRF
 t/pod-coverage.t
 t/pod.t
-conf/input/isis.ini
 conf/lookup/example.pm
 conf/lookup/isis.pm
 conf/normalize/isis.xml

diff --git a/Makefile.PL b/Makefile.PL
index 81a97c0..93122da 100644 (file)
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -53,11 +53,14 @@ tags:
 sf:
        svn2cvs.pl file:///home/dpavlin/private/svn/webpac2/trunk/ :ext:dpavlin@cvs.sourceforge.net:/cvsroot/webpac webpac2
 
-run:
-       rm -f log
+config_yml:
        test ! -e conf/config.yml && ln -s /data/Webpacus/config.yml conf/ || true
-       test -d db && rm -Rf db/* || mkdir db
-       ./run.pl
+
+run: config_yml
+       ./run.pl --clean --limit 100
+
+index: config_yml
+       ./run.pl --clean
 
 MAKE_MORE
 }

diff --git a/lib/WebPAC/Input.pm b/lib/WebPAC/Input.pm
index 52494ac..416844e 100644 (file)
--- a/lib/WebPAC/Input.pm
+++ b/lib/WebPAC/Input.pm
@@ -101,8 +101,7 @@ sub new {
                my $n = $self->{module} . '::' . $subclass;
                if (! defined &{ $n }) {
                        my $missing = "missing $subclass in $self->{module}";
-                       $log->logwarn($missing);
-                       $self->{$subclass} = sub { warn "$missing\n" };
+                       $self->{$subclass} = sub { $log->logwarn($missing) };
                } else {
                        $self->{$subclass} = \&{ $n };
                }
@@ -210,7 +209,7 @@ sub open {
        }
 
        if ($self->{limit}) {
-               $log->info("limiting to ",$self->{limit}," records");
+               $log->debug("limiting to ",$self->{limit}," records");
                $limit = $offset + $self->{limit} - 1;
                $limit = $size if ($limit > $size);
        }

diff --git a/lib/WebPAC/Output/Estraier.pm b/lib/WebPAC/Output/Estraier.pm
index 5ca1b90..bafa037 100644 (file)
--- a/lib/WebPAC/Output/Estraier.pm
+++ b/lib/WebPAC/Output/Estraier.pm
@@ -90,14 +90,14 @@ sub new {
        my $url = $self->{masterurl} . '/node/' . $self->{database};
        $self->{url} = $url;
 
-       $log->info("opening Hyper Estraier index $self->{url}");
+       $log->debug("opening index $self->{url}");
 
        my $nodes = $self->master( action => 'nodelist' );
 
        $log->debug("nodes found: $nodes");
 
        if ($nodes !~ m/^$self->{database}\t/sm) {
-               $log->info("creating index $url");
+               $log->warn("creating index $url");
                $self->master(
                        action => 'nodeadd',
                        name => $self->{database},
@@ -109,7 +109,7 @@ sub new {
        $self->{'db'}->set_auth($self->{'user'}, $self->{passwd});
 
        my $encoding = $self->{'encoding'} || 'ISO-8859-2';
-       $log->info("using encoding $encoding");
+       $log->info("using index $self->{url} with encoding $encoding");
 
        $self->{'iconv'} = new Text::Iconv($encoding, 'UTF-8') or
                $log->logdie("can't create conversion from $encoding to UTF-8");

diff --git a/run.pl b/run.pl
index f69d51f..e5bcdad 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -15,13 +15,57 @@ use WebPAC::Normalize::XML;
 use WebPAC::Output::TT;
 use WebPAC::Output::Estraier 0.05;
 use YAML qw/LoadFile/;
-use LWP::Simple;
+use Getopt::Long;
+use File::Path;
 
-my $limit = shift @ARGV;
+=head1 NAME
 
-my $config = LoadFile('conf/config.yml');
+run.pl - start WebPAC indexing
 
-print "config = ",Dumper($config);
+B<this command will probably go away. Don't get used to it!>
+
+Options:
+
+=over 4
+
+=item --offset 42
+
+start loading (all) databases at offset 42
+
+=item --limit 100
+
+limit loading to 100 records
+
+=item --clean
+
+remove database and Hyper Estraier index before indexing
+
+=item --config conf/config.yml
+
+path to YAML configuration file
+
+=back
+
+=cut
+
+my $offset;
+my $limit;
+
+my $clean = 0;
+my $config = 'conf/config.yml';
+my $debug = 0;
+
+GetOptions(
+       "limit=i" => \$limit,
+       "offset=i" => \$offset,
+       "clean" => \$clean,
+       "config" => \$config,
+       "debug" => \$debug,
+);
+
+$config = LoadFile($config);
+
+print "config = ",Dumper($config) if ($debug);
 
 die "no databases in config file!\n" unless ($config->{databases});
 
@@ -38,12 +82,16 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
        my $est_config = $config->{hyperestraier} || $log->logdie("can't find 'hyperestraier' part in confguration");
        $est_config->{database} = $database;
 
-       $log->info("using HyperEstraier URL $est_config->{masterurl}");
-
        my $est = new WebPAC::Output::Estraier(
                %{ $est_config },
        );
 
+       if ($clean) {
+               $log->warn("creating new empty index $database");
+               $est->master( action => 'nodedel', name => $database );
+               $est->master( action => 'nodeadd', name => $database, label => $database );
+       }
+
        #
        # now WebPAC::Store
        #
@@ -52,12 +100,17 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
 
        my $db_path = $config->{webpac}->{db_path} . '/' . $database;
 
-       $log->info("working on $database in $db_path");
+       if ($clean) {
+               $log->info("creating new database $database in $db_path");
+               rmtree( $db_path ) || $log->warn("can't remove $db_path: $!");
+       } else {
+               $log->info("working on $database in $db_path");
+       }
 
        my $db = new WebPAC::Store(
                path => $db_path,
                database => $database,
-               debug => 1,
+               debug => $debug,
        );
 
 
@@ -93,7 +146,8 @@ while (my ($database, $db_config) = each %{ $config->{databases} }) {
                my $input_db = new WebPAC::Input(
                        module => $input_module,
                        code_page => $config->{webpac}->{webpac_encoding},
-                       limit => $input->{limit},
+                       limit => $limit || $input->{limit},
+                       offset => $offset,
                        lookup => $lookup,
                );
                $log->logdie("can't create input using $input_module") unless ($input);