X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=misc%2Fbin%2Fzebraqueue_daemon.pl;h=6181a94c03f8d32b79c88029b3639f858ad8f17f;hb=f57ceeb8c56cfcae782db9802f5f6df8a2a1bc93;hp=ea788d5ff55958e57a54622327b632dfa4da54c3;hpb=c82afab40a13068b7edd1d31dc737315f1ca9bd9;p=koha.git diff --git a/misc/bin/zebraqueue_daemon.pl b/misc/bin/zebraqueue_daemon.pl index ea788d5ff5..6181a94c03 100755 --- a/misc/bin/zebraqueue_daemon.pl +++ b/misc/bin/zebraqueue_daemon.pl @@ -3,6 +3,7 @@ # daemon to watch the zebraqueue and update zebra as needed use strict; +#use warnings; FIXME - Bug 2505 BEGIN { # find Koha's Perl modules # test carefully before changing this @@ -110,7 +111,7 @@ sub handler_sleep { # can be used to slow down loop execution if needed my ( $kernel, $heap, $session ) = @_[ KERNEL, HEAP, SESSION ]; use Time::HiRes qw (sleep); - Time::HiRes::sleep(0.01); + Time::HiRes::sleep(0.5); #sleep 1; $kernel->yield('status_check'); } @@ -125,10 +126,14 @@ sub handler_check { if ($data->{'opcount'} > 0) { Unix::Syslog::syslog LOG_INFO, "$data->{'opcount'} operations waiting to be run\n"; $sth->finish(); + $dbh->commit(); # needed so that we get current state of zebraqueue next time + # we enter handler_check $kernel->yield('do_ops'); } else { $sth->finish(); + $dbh->commit(); # needed so that we get current state of zebraqueue next time + # we enter handler_check $kernel->yield('sleep'); } } @@ -137,10 +142,11 @@ sub zebraop { # execute operations waiting in the zebraqueue my ( $kernel, $heap, $session ) = @_[ KERNEL, HEAP, SESSION ]; my $dbh = get_db_connection(); - my $readsth = $dbh->prepare("SELECT id, biblio_auth_number, operation, server FROM zebraqueue WHERE done = 0"); + my $readsth = $dbh->prepare("SELECT id, biblio_auth_number, operation, server FROM zebraqueue WHERE done = 0 ORDER BY id DESC"); $readsth->execute(); Unix::Syslog::syslog LOG_INFO, "Executing zebra operations\n"; + my $completed_updates = {}; ZEBRAQUEUE: while (my $data = $readsth->fetchrow_hashref()) { warn "Inside while loop" if $debug; @@ -152,6 +158,7 @@ sub zebraop { my $server = $data->{'server'}; next ZEBRAQUEUE if exists $postponed_updates->{$server}->{$record_number}; + next ZEBRAQUEUE if exists $completed_updates->{$server}->{$record_number}->{$op}; my $ok = 0; my $record; @@ -162,11 +169,16 @@ sub zebraop { $ok = process_update($dbh, $server, $record_number, $id); } if ($ok == 1) { - mark_done($dbh, $record_number, $op); + mark_done($dbh, $record_number, $op, $server); + $completed_updates->{$server}->{$record_number}->{$op} = 1; + if ($op eq 'recordDelete') { + $completed_updates->{$server}->{$record_number}->{'specialUpdate'} = 1; + } } } $readsth->finish(); - $kernel->yield('status_check'); + $dbh->commit(); + $kernel->yield('sleep'); } sub process_delete { @@ -190,8 +202,8 @@ sub process_delete { $ok = 1; } else { # caught a ZOOM::Exception - my $error = _format_zoom_error_message($@); - warn "ERROR: $error"; + my $message = _format_zoom_error_message($@); + postpone_update($server, $record_number, $message); } } else { # then, delete the record @@ -227,7 +239,7 @@ sub process_update { ## it's Broken XML-- Should not reach here-- but if it does -lets protect ZEBRA if ($@) { Unix::Syslog::syslog LOG_ERR, "$server record $record_number is malformed: $@"; - mark_done_by_id($dbh, $id); + mark_done_by_id($dbh, $id, $server); $ok = 0; } else { # ok, we have everything, do the operation in zebra ! @@ -239,14 +251,16 @@ sub process_update { sub mark_done_by_id { my $dbh = shift; my $id = shift; - my $delsth = $dbh->prepare("UPDATE zebraqueue SET done = 1 WHERE id = ?"); - $delsth->execute($id); + my $server = shift; + my $delsth = $dbh->prepare("UPDATE zebraqueue SET done = 1 WHERE id = ? AND server = ? AND done = 0"); + $delsth->execute($id, $server); } sub mark_done { my $dbh = shift; my $record_number = shift; my $op = shift; + my $server = shift; my $delsth; if ($op eq 'recordDelete') { @@ -254,15 +268,21 @@ sub mark_done { # did a modif (or item deletion) just before biblio deletion, there are some specialUpdate # that are pending and can't succeed, as we don't have the XML anymore # so, delete everything for this biblionumber - $delsth = $dbh->prepare_cached("UPDATE zebraqueue SET done=1 WHERE biblio_auth_number = ? and operation = ?"); - $delsth->execute($record_number, $op); + $delsth = $dbh->prepare_cached("UPDATE zebraqueue SET done = 1 + WHERE biblio_auth_number = ? + AND server = ? + AND done = 0"); + $delsth->execute($record_number, $server); } else { # if it's not a deletion, delete every pending specialUpdate for this biblionumber # in case the user add biblio, then X items, before this script runs # this avoid indexing X+1 times where just 1 is enough. $delsth = $dbh->prepare("UPDATE zebraqueue SET done = 1 - WHERE biblio_auth_number = ? and operation = 'specialUpdate'"); - $delsth->execute($record_number); + WHERE biblio_auth_number = ? + AND operation = 'specialUpdate' + AND server = ? + AND done = 0"); + $delsth->execute($record_number, $server); } } @@ -308,6 +328,9 @@ sub zebrado { postpone_update($server, $record_number, $message); } } + # FIXME - would be more efficient to send a ES commit + # after a batch of records, rather than commiting after + # each one - Zebra handles updates relatively slowly. eval { $Zpackage->send('commit'); }; if ($@) { # operation succeeded, but commit @@ -334,6 +357,14 @@ sub postpone_update { $message .= "\n" unless $message =~ /\n$/; Unix::Syslog::syslog LOG_ERR, $message; $postponed_updates->{$server}->{$record_number} = 1; + + $num_postponed_updates++; + if ($num_postponed_updates > $max_postponed_updates) { + warn "exiting, over $max_postponed_updates postponed indexing updates"; + Unix::Syslog::syslog LOG_ERR, "exiting, over $max_postponed_updates postponed indexing updates"; + Unix::Syslog::closelog; + exit; + } } sub handler_stop { @@ -362,6 +393,8 @@ sub get_db_connection { # C4::Context->dbh dies if it cannot # establish a connection $db_connection_wait = $min_connection_wait; + $dbh->{AutoCommit} = 0; # do this to reduce number of + # commits to zebraqueue return $dbh; }