Turn on option low_mem (which need rewrite to use db/row) if there

are more than 10000 rows (hardcoded, but should go away).

This prevents webpac from running out of memory with databases
of about 300000 records on 4Gb of (virtual) memory.

git-svn-id: svn+ssh://mjesec/home/dpavlin/svn/webpac2/trunk@760 07558da8-63fa-0310-ba24-9fe276d99e06

diff --git a/lib/WebPAC/Input.pm b/lib/WebPAC/Input.pm
index 21bea8f..cab03cc 100644 (file)
--- a/lib/WebPAC/Input.pm
+++ b/lib/WebPAC/Input.pm
@@ -112,32 +112,6 @@ sub new {
 
        $self->{'encoding'} ||= 'ISO-8859-2';
 
-       # running with low_mem flag? well, use DBM::Deep then.
-       if ($self->{'low_mem'}) {
-               $log->info("running with low_mem which impacts performance (<32 Mb memory usage)");
-
-               my $db_file = "data.db";
-
-               if (-e $db_file) {
-                       unlink $db_file or $log->logdie("can't remove '$db_file' from last run");
-                       $log->debug("removed '$db_file' from last run");
-               }
-
-               require DBM::Deep;
-
-               my $db = new DBM::Deep $db_file;
-
-               $log->logdie("DBM::Deep error: $!") unless ($db);
-
-               if ($db->error()) {
-                       $log->logdie("can't open '$db_file' under low_mem: ",$db->error());
-               } else {
-                       $log->debug("using file '$db_file' for DBM::Deep");
-               }
-
-               $self->{'db'} = $db;
-       }
-
        $self ? return $self : return undef;
 }
 
@@ -285,6 +259,38 @@ sub open {
 
        $log->info("processing $self->{size}/$size records [$from_rec-$to_rec] convert $code_page -> $self->{encoding}", $self->{stats} ? ' [stats]' : '');
 
+       # turn on low_mem for databases with more than 100000 records!
+       if (! $self->{low_mem} && $size > 100000) {
+               $log->warn("Using on-disk storage instead of memory for input data. This will affect performance.");
+               $self->{low_mem}++;
+       }
+
+       # running with low_mem flag? well, use DBM::Deep then.
+       if ($self->{'low_mem'}) {
+               $log->info("running with low_mem which impacts performance (<32 Mb memory usage)");
+
+               my $db_file = "data.db";
+
+               if (-e $db_file) {
+                       unlink $db_file or $log->logdie("can't remove '$db_file' from last run");
+                       $log->debug("removed '$db_file' from last run");
+               }
+
+               require DBM::Deep;
+
+               my $db = new DBM::Deep $db_file;
+
+               $log->logdie("DBM::Deep error: $!") unless ($db);
+
+               if ($db->error()) {
+                       $log->logdie("can't open '$db_file' under low_mem: ",$db->error());
+               } else {
+                       $log->debug("using file '$db_file' for DBM::Deep");
+               }
+
+               $self->{'db'} = $db;
+       }
+
        # read database
        for (my $pos = $from_rec; $pos <= $to_rec; $pos++) {