speed boost: command-line stage and commit biblios

Turned off autocommit and commit every 100 records.

Signed-off-by: Chris Cormack <crc@liblime.com>
Signed-off-by: Joshua Ferraro <jmf@liblime.com>

diff --git a/misc/commit_biblios_file.pl b/misc/commit_biblios_file.pl
index 4c8d8ac..2f9851f 100755 (executable)
--- a/misc/commit_biblios_file.pl
+++ b/misc/commit_biblios_file.pl
@@ -39,12 +39,15 @@ if ($list_batches) {
 # in future, probably should tie to a real user account
 C4::Context->set_userenv(0, 'batch', 0, 'batch', 'batch', 'batch', 'batch', 'batch');
 
+my $dbh = C4::Context->dbh;
+$dbh->{AutoCommit} = 0;
 if ($batch_number =~ /^\d+$/ and $batch_number > 0) {
     my $batch = GetImportBatch($batch_number);
     die "$0: import batch $batch_number does not exist in database\n" unless defined $batch;
     die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n"
         unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted";
     process_batch($batch_number);
+    $dbh->commit();
 } else {
     die "$0: please specify a numeric batch ID\n";
 }
@@ -71,7 +74,7 @@ sub process_batch {
 
     print "... importing MARC records -- please wait\n";
     my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = 
-        BatchCommitBibRecords($import_batch_id, 100, \&print_progress);
+        BatchCommitBibRecords($import_batch_id, 100, \&print_progress_and_commit);
     print "... finished importing MARC records\n";
 
     print <<_SUMMARY_;
@@ -90,9 +93,10 @@ duplicate of one already in the database.
 _SUMMARY_
 }
 
-sub print_progress {
+sub print_progress_and_commit {
     my $recs = shift;
     print "... processed $recs records\n";
+    $dbh->commit();
 }
 
 sub print_usage {

diff --git a/misc/stage_biblios_file.pl b/misc/stage_biblios_file.pl
index bfe26ce..a692748 100755 (executable)
--- a/misc/stage_biblios_file.pl
+++ b/misc/stage_biblios_file.pl
@@ -39,7 +39,10 @@ unless (-r $input_file) {
     die "$0: cannot open input file $input_file: $!\n";
 }
 
+my $dbh = C4::Context->dbh;
+$dbh->{AutoCommit} = 0;
 process_batch($input_file, $match_bibs, $add_items, $batch_comment);
+$dbh->commit();
 
 exit 0;
 
@@ -62,7 +65,7 @@ sub process_batch {
     print "... staging MARC records -- please wait\n";
     my ($batch_id, $num_valid, $num_items, @import_errors) = 
         BatchStageMarcRecords($marc_flavor, $marc_records, $input_file, $batch_comment, '', $add_items, 0,
-                              100, \&print_progress);
+                              100, \&print_progress_and_commit);
     print "... finished staging MARC records\n";
 
     my $num_with_matches = 0;
@@ -72,7 +75,7 @@ sub process_batch {
         $matcher->add_simple_required_check('245', 'a', -1, 0, '', 
                                             '245', 'a', -1, 0, '');
         print "... looking for matches with records already in database\n";
-        $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress);
+        $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit);
         print "... finished looking for matches\n";
     }
 
@@ -102,8 +105,9 @@ _SUMMARY_
     print "\n";
 }
 
-sub print_progress {
+sub print_progress_and_commit {
     my $recs = shift;
+    $dbh->commit();
     print "... processed $recs records\n";
 }