From bb81fce73e4f5ec9c43cfb21e594acfa192b9638 Mon Sep 17 00:00:00 2001
From: Dobrica Pavlinusic <dpavlin@rot13.org>
Date: Wed, 25 Oct 2006 15:56:44 +0000
Subject: [PATCH] Turn on option low_mem (which need rewrite to use db/row) if
 there are more than 10000 rows (hardcoded, but should go away).

This prevents webpac from running out of memory with databases
of about 300000 records on 4Gb of (virtual) memory.


git-svn-id: svn+ssh://mjesec/home/dpavlin/svn/webpac2/trunk@760 07558da8-63fa-0310-ba24-9fe276d99e06
---
 lib/WebPAC/Input.pm | 58 +++++++++++++++++++++++++--------------------
 1 file changed, 32 insertions(+), 26 deletions(-)

diff --git a/lib/WebPAC/Input.pm b/lib/WebPAC/Input.pm
index 21bea8f..cab03cc 100644
--- a/lib/WebPAC/Input.pm
+++ b/lib/WebPAC/Input.pm
@@ -112,32 +112,6 @@ sub new {
 
 	$self->{'encoding'} ||= 'ISO-8859-2';
 
-	# running with low_mem flag? well, use DBM::Deep then.
-	if ($self->{'low_mem'}) {
-		$log->info("running with low_mem which impacts performance (<32 Mb memory usage)");
-
-		my $db_file = "data.db";
-
-		if (-e $db_file) {
-			unlink $db_file or $log->logdie("can't remove '$db_file' from last run");
-			$log->debug("removed '$db_file' from last run");
-		}
-
-		require DBM::Deep;
-
-		my $db = new DBM::Deep $db_file;
-
-		$log->logdie("DBM::Deep error: $!") unless ($db);
-
-		if ($db->error()) {
-			$log->logdie("can't open '$db_file' under low_mem: ",$db->error());
-		} else {
-			$log->debug("using file '$db_file' for DBM::Deep");
-		}
-
-		$self->{'db'} = $db;
-	}
-
 	$self ? return $self : return undef;
 }
 
@@ -285,6 +259,38 @@ sub open {
 
 	$log->info("processing $self->{size}/$size records [$from_rec-$to_rec] convert $code_page -> $self->{encoding}", $self->{stats} ? ' [stats]' : '');
 
+	# turn on low_mem for databases with more than 100000 records!
+	if (! $self->{low_mem} && $size > 100000) {
+		$log->warn("Using on-disk storage instead of memory for input data. This will affect performance.");
+		$self->{low_mem}++;
+	}
+
+	# running with low_mem flag? well, use DBM::Deep then.
+	if ($self->{'low_mem'}) {
+		$log->info("running with low_mem which impacts performance (<32 Mb memory usage)");
+
+		my $db_file = "data.db";
+
+		if (-e $db_file) {
+			unlink $db_file or $log->logdie("can't remove '$db_file' from last run");
+			$log->debug("removed '$db_file' from last run");
+		}
+
+		require DBM::Deep;
+
+		my $db = new DBM::Deep $db_file;
+
+		$log->logdie("DBM::Deep error: $!") unless ($db);
+
+		if ($db->error()) {
+			$log->logdie("can't open '$db_file' under low_mem: ",$db->error());
+		} else {
+			$log->debug("using file '$db_file' for DBM::Deep");
+		}
+
+		$self->{'db'} = $db;
+	}
+
 	# read database
 	for (my $pos = $from_rec; $pos <= $to_rec; $pos++) {
 
-- 
2.20.1