package C4::Matcher;
-# Copyright (C) 2007 LibLime
+# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
#
# This file is part of Koha.
#
-# Koha is free software; you can redistribute it and/or modify it under the
-# terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
#
-# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
-# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
#
-# You should have received a copy of the GNU General Public License along with
-# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
-# Suite 330, Boston, MA 02111-1307 USA
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>.
-use strict;
-use C4::Context;
-use MARC::Record;
-use C4::Search;
-use C4::Biblio;
+use Modern::Perl;
-use vars qw($VERSION);
+use MARC::Record;
-BEGIN {
- # set the version for version checking
- $VERSION = 3.01;
-}
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
+use Koha::SearchEngine::QueryBuilder;
+use Koha::Util::Normalize qw/legacy_default remove_spaces upper_case lower_case/;
=head1 NAME
=head1 SYNOPSIS
-=over 4
+ my @matchers = C4::Matcher::GetMatcherList();
-my @matchers = C4::Matcher::GetMatcherList();
+ my $matcher = C4::Matcher->new($record_type);
+ $matcher->threshold($threshold);
+ $matcher->code($code);
+ $matcher->description($description);
-my $matcher = C4::Matcher->new($record_type);
-$matcher->threshold($threshold);
-$matcher->code($code);
-$matcher->description($description);
+ $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
+ $matcher->add_simple_matchpoint('Date', 1000, '008', '', 7, 4, '');
+ $matcher->add_matchpoint('isbn', 1000, [ { tag => '020', subfields => 'a', norms => [] } ]);
-$matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
-$matcher->add_simple_matchpoint('Date', 1000, '008', '', 7, 4, '');
-$matcher->add_matchpoint('isbn', 1000, [ { tag => '020', subfields => 'a', norms => [] } ]);
+ $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, '');
+ $matcher->add_required_check([ { tag => '245', subfields => 'a', norms => [] } ],
+ [ { tag => '245', subfields => 'a', norms => [] } ]);
-$matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, '');
-$matcher->add_required_check([ { tag => '245', subfields => 'a', norms => [] } ],
- [ { tag => '245', subfields => 'a', norms => [] } ]);
+ my @matches = $matcher->get_matches($marc_record, $max_matches);
-my @matches = $matcher->get_matches($marc_record, $max_matches);
+ foreach $match (@matches) {
-foreach $match (@matches) {
+ # matches already sorted in order of
+ # decreasing score
+ print "record ID: $match->{'record_id'};
+ print "score: $match->{'score'};
- # matches already sorted in order of
- # decreasing score
- print "record ID: $match->{'record_id'};
- print "score: $match->{'score'};
+ }
-}
-
-my $matcher_description = $matcher->dump();
-
-=back
+ my $matcher_description = $matcher->dump();
=head1 FUNCTIONS
=head2 GetMatcherList
-=over 4
-
-my @matchers = C4::Matcher::GetMatcherList();
-
-=back
+ my @matchers = C4::Matcher::GetMatcherList();
Returns an array of hashrefs list all matchers
present in the database. Each hashref includes:
-matcher_id
-code
-description
+ * matcher_id
+ * code
+ * description
=cut
return @results;
}
-=head1 METHODS
+=head2 GetMatcherId
+
+ my $matcher_id = C4::Matcher::GetMatcherId($code);
+
+Returns the matcher_id of a code.
=cut
-=head2 new
+sub GetMatcherId {
+ my ($code) = @_;
+ my $dbh = C4::Context->dbh;
-=over 4
+ my $matcher_id = $dbh->selectrow_array("SELECT matcher_id FROM marc_matchers WHERE code = ?", undef, $code);
+ return $matcher_id;
+}
-my $matcher = C4::Matcher->new($record_type, $threshold);
+=head1 METHODS
-=back
+=head2 new
+
+ my $matcher = C4::Matcher->new($record_type, $threshold);
Creates a new Matcher. C<$record_type> indicates which search
database to use, e.g., 'biblio' or 'authority' and defaults to
=head2 fetch
-=over 4
-
-my $matcher = C4::Matcher->fetch($id);
-
-=back
+ my $matcher = C4::Matcher->fetch($id);
Creates a matcher object from the version stored
in the database. If a matcher with the given
=head2 store
-=over 4
-
-my $id = $matcher->store();
-
-=back
+ my $id = $matcher->store();
Stores matcher in database. The return value is the ID
of the marc_matchers row. If the matcher was
my $matcher_id = $self->{'id'};
$sth = $dbh->prepare_cached("INSERT INTO matchpoints (matcher_id, search_index, score)
VALUES (?, ?, ?)");
- $sth->execute($matcher_id, $matchpoint->{'index'}, $matchpoint->{'score'});
+ $sth->execute($matcher_id, $matchpoint->{'index'}, $matchpoint->{'score'}||0);
my $matchpoint_id = $dbh->{'mysql_insertid'};
my $seqnum = 0;
foreach my $component (@{ $matchpoint->{'components'} }) {
$sth->bind_param(2, $seqnum);
$sth->bind_param(3, $component->{'tag'});
$sth->bind_param(4, join "", sort keys %{ $component->{'subfields'} });
- $sth->bind_param(5, $component->{'offset'});
+ $sth->bind_param(5, $component->{'offset'}||0);
$sth->bind_param(6, $component->{'length'});
$sth->execute();
my $matchpoint_component_id = $dbh->{'mysql_insertid'};
=head2 delete
-=over 4
-
-C4::Matcher->delete($id);
-
-=back
+ C4::Matcher->delete($id);
Deletes the matcher of the specified ID
from the database.
$sth->execute($matcher_id); # relying on cascading deletes to clean up everything
}
-=head2 threshold
+=head2 record_type
-=over 4
+ $matcher->record_type('biblio');
+ my $record_type = $matcher->record_type();
-$matcher->threshold(1000);
-my $threshold = $matcher->threshold();
+Accessor method.
-=back
+=cut
+
+sub record_type {
+ my $self = shift;
+ @_ ? $self->{'record_type'} = shift : $self->{'record_type'};
+}
+
+=head2 threshold
+
+ $matcher->threshold(1000);
+ my $threshold = $matcher->threshold();
Accessor method.
=head2 _id
-=over 4
-
-$matcher->_id(123);
-my $id = $matcher->_id();
-
-=back
+ $matcher->_id(123);
+ my $id = $matcher->_id();
Accessor method. Note that using this method
to set the DB ID of the matcher should not be
=head2 code
-=over 4
-
-$matcher->code('ISBN');
-my $code = $matcher->code();
-
-=back
+ $matcher->code('ISBN');
+ my $code = $matcher->code();
Accessor method.
=head2 description
-=over 4
-
-$matcher->description('match on ISBN');
-my $description = $matcher->description();
-
-=back
+ $matcher->description('match on ISBN');
+ my $description = $matcher->description();
Accessor method.
=head2 add_matchpoint
-=over 4
-
-$matcher->add_matchpoint($index, $score, $matchcomponents);
-
-=back
+ $matcher->add_matchpoint($index, $score, $matchcomponents);
Adds a matchpoint that may include multiple components. The $index
parameter identifies the index that will be searched, while $score
=head2 add_simple_matchpoint
-=over 4
+ $matcher->add_simple_matchpoint($index, $score, $source_tag,
+ $source_subfields, $source_offset,
+ $source_length, $source_normalizer);
-$matcher->add_simple_matchpoint($index, $score, $source_tag, $source_subfields,
- $source_offset, $source_length,
- $source_normalizer);
-
-=back
Adds a simple matchpoint rule -- after composing a key based on the source tag and subfields,
normalized per the normalization fuction, search the index. All records retrieved
=head2 add_required_check
-=over 4
-
-$match->add_required_check($source_matchpoint, $target_matchpoint);
-
-=back
+ $match->add_required_check($source_matchpoint, $target_matchpoint);
Adds a required check definition. A required check means that in
order for a match to be considered valid, the key derived from the
=head2 add_simple_required_check
-$matcher->add_simple_required_check($source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer,
- $target_tag, $target_subfields, $target_offset, $target_length, $target_normalizer);
-
-=over 4
+ $matcher->add_simple_required_check($source_tag, $source_subfields,
+ $source_offset, $source_length, $source_normalizer,
+ $target_tag, $target_subfields, $target_offset,
+ $target_length, $target_normalizer);
Adds a required check, which requires that the normalized keys made from the source and targets
must match for a match to be considered valid.
-=back
-
=cut
sub add_simple_required_check {
);
}
-=head2 find_matches
-
-=over 4
-
-my @matches = $matcher->get_matches($marc_record, $max_matches);
-foreach $match (@matches) {
- # matches already sorted in order of
- # decreasing score
- print "record ID: $match->{'record_id'};
- print "score: $match->{'score'};
-}
+=head2 get_matches
-=back
+ my @matches = $matcher->get_matches($marc_record, $max_matches);
+ foreach $match (@matches) {
+ # matches already sorted in order of
+ # decreasing score
+ print "record ID: $match->{'record_id'};
+ print "score: $match->{'score'};
+ }
Identifies all of the records matching the given MARC record. For a record already
in the database to be considered a match, it must meet the following criteria:
my $self = shift;
my ($source_record, $max_matches) = @_;
- my %matches = ();
+ my $matches = {};
+ my $marcframework_used = ''; # use the default framework
+
+ my $QParser;
+ $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
+ foreach my $matchpoint ( @{ $self->{'matchpoints'} } ) {
+ my @source_keys = _get_match_keys( $source_record, $matchpoint );
- foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
- my @source_keys = _get_match_keys($source_record, $matchpoint);
next if scalar(@source_keys) == 0;
+
+ # FIXME - because of a bug in QueryParser, an expression ofthe
+ # format 'isbn:"isbn1" || isbn:"isbn2" || isbn"isbn3"...'
+ # does not get parsed correctly, so we will not
+ # do AggressiveMatchOnISBN if UseQueryParser is on
+ @source_keys = C4::Koha::GetVariationsOfISBNs(@source_keys)
+ if ( $matchpoint->{index} =~ /^isbn$/i
+ && C4::Context->preference('AggressiveMatchOnISBN') )
+ && !C4::Context->preference('UseQueryParser');
+
+ @source_keys = C4::Koha::GetVariationsOfISSNs(@source_keys)
+ if ( $matchpoint->{index} =~ /^issn$/i
+ && C4::Context->preference('AggressiveMatchOnISSN') )
+ && !C4::Context->preference('UseQueryParser');
+
# build query
- my $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys);
- # FIXME only searching biblio index at the moment
- my ($error, $searchresults, $total_hits) = SimpleSearch($query, 0, $max_matches);
+ my $query;
+ my $error;
+ my $searchresults;
+ my $total_hits;
+ if ( $self->{'record_type'} eq 'biblio' ) {
+
+ #NOTE: The QueryParser can't handle the CCL syntax of 'qualifier','qualifier', so fallback to non-QueryParser.
+ #NOTE: You can see this in C4::Search::SimpleSearch() as well in a different way.
+ if ($QParser && $matchpoint->{'index'} !~ m/\w,\w/) {
+ $query = join( " || ",
+ map { "$matchpoint->{'index'}:$_" } @source_keys );
+ }
+ else {
+ my $phr = ( C4::Context->preference('AggressiveMatchOnISBN') || C4::Context->preference('AggressiveMatchOnISSN') ) ? ',phr' : q{};
+ $query = join( " or ",
+ map { "$matchpoint->{'index'}$phr=\"$_\"" } @source_keys );
+ #NOTE: double-quote the values so you don't get a "Embedded truncation not supported" error when a term has a ? in it.
+ }
- warn "search failed ($query) $error" if $error;
- foreach my $matched (@$searchresults) {
- $matches{$matched} += $matchpoint->{'score'};
+ # Use state variables to avoid recreating the objects every time.
+ # With Elasticsearch this also avoids creating a massive amount of
+ # ES connectors that would eventually run out of file descriptors.
+ state $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
+ ( $error, $searchresults, $total_hits ) =
+ $searcher->simple_search_compat( $query, 0, $max_matches, undef, skip_normalize => 1 );
+
+ if ( defined $error ) {
+ warn "search failed ($query) $error";
+ }
+ else {
+ if ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' ) {
+ foreach my $matched ( @{$searchresults} ) {
+ my ( $biblionumber_tag, $biblionumber_subfield ) = C4::Biblio::GetMarcFromKohaField( "biblio.biblionumber", $marcframework_used );
+ my $id = ( $biblionumber_tag > 10 ) ?
+ $matched->field($biblionumber_tag)->subfield($biblionumber_subfield) :
+ $matched->field($biblionumber_tag)->data();
+ $matches->{$id}->{score} += $matchpoint->{score};
+ $matches->{$id}->{record} = $matched;
+ }
+ }
+ else {
+ foreach my $matched ( @{$searchresults} ) {
+ $matches->{$matched}->{score} += $matchpoint->{'score'};
+ $matches->{$matched}->{record} = $matched;
+ }
+ }
+ }
+
+ }
+ elsif ( $self->{'record_type'} eq 'authority' ) {
+ my @marclist;
+ my @and_or;
+ my @excluding = [];
+ my @operator;
+ my @value;
+ foreach my $key (@source_keys) {
+ push @marclist, $matchpoint->{'index'};
+ push @and_or, 'or';
+ push @operator, 'exact';
+ push @value, $key;
+ }
+ # Use state variables to avoid recreating the objects every time.
+ # With Elasticsearch this also avoids creating a massive amount of
+ # ES connectors that would eventually run out of file descriptors.
+ state $builder = Koha::SearchEngine::QueryBuilder->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
+ state $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
+ my $search_query = $builder->build_authorities_query_compat(
+ \@marclist, \@and_or, \@excluding, \@operator,
+ \@value, undef, 'AuthidAsc'
+ );
+ my ( $authresults, $total ) = $searcher->search_auth_compat( $search_query, 0, 20 );
+
+ foreach my $result (@$authresults) {
+ my $id = $result->{authid};
+ $matches->{$id}->{score} += $matchpoint->{'score'};
+ $matches->{$id}->{record} = $id;
+ }
}
}
# get rid of any that don't meet the threshold
- %matches = map { ($matches{$_} >= $self->{'threshold'}) ? ($_ => $matches{$_}) : () } keys %matches;
-
- # get rid of any that don't meet the required checks
- %matches = map { _passes_required_checks($source_record, $_, $self->{'required_checks'}) ? ($_ => $matches{$_}) : () }
- keys %matches;
+ $matches = { map { ($matches->{$_}->{score} >= $self->{'threshold'}) ? ($_ => $matches->{$_}) : () } keys %$matches };
my @results = ();
- foreach my $marcblob (keys %matches) {
- my $target_record = MARC::Record->new_from_usmarc($marcblob);
- my $result = TransformMarcToKoha(C4::Context->dbh, $target_record, '');
- # FIXME - again, bibliospecific
- # also, can search engine be induced to give just the number in the first place?
- my $record_number = $result->{'biblionumber'};
- push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} };
+ if ($self->{'record_type'} eq 'biblio') {
+ require C4::Biblio;
+ # get rid of any that don't meet the required checks
+ $matches = {
+ map {
+ _passes_required_checks( $source_record, $_, $self->{'required_checks'} )
+ ? ( $_ => $matches->{$_} )
+ : ()
+ } keys %$matches
+ };
+
+ foreach my $id ( keys %$matches ) {
+ my $target_record = C4::Search::new_record_from_zebra( 'biblioserver', $matches->{$id}->{record} );
+ my $result = C4::Biblio::TransformMarcToKoha( $target_record, $marcframework_used );
+ push @results, {
+ record_id => $result->{biblionumber},
+ score => $matches->{$id}->{score}
+ };
+ }
+ } elsif ($self->{'record_type'} eq 'authority') {
+ require C4::AuthoritiesMarc;
+ foreach my $id (keys %$matches) {
+ push @results, {
+ record_id => $id,
+ score => $matches->{$id}->{score}
+ };
+ }
}
- @results = sort { $b->{'score'} cmp $a->{'score'} } @results;
+ @results = sort {
+ $b->{'score'} cmp $a->{'score'} or
+ $b->{'record_id'} cmp $a->{'record_id'}
+ } @results;
if (scalar(@results) > $max_matches) {
@results = @results[0..$max_matches-1];
}
return @results;
-
}
=head2 dump
-=over 4
-
-$description = $matcher->dump();
-
-=back
+ $description = $matcher->dump();
Returns a reference to a structure containing all of the information
in the matcher object. This is mainly a convenience method to
$result->{'matcher_id'} = $self->{'id'};
$result->{'code'} = $self->{'code'};
$result->{'description'} = $self->{'description'};
+ $result->{'record_type'} = $self->{'record_type'};
$result->{'matchpoints'} = [];
foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
}
sub _get_match_keys {
+
my $source_record = shift;
my $matchpoint = shift;
my $check_only_first_repeat = @_ ? shift : 0;
# If there are two 003s and two 001s, there will be two keys:
# first 003 + first 001
# second 003 + second 001
-
+
my @keys = ();
for (my $i = 0; $i <= $#{ $matchpoint->{'components'} }; $i++) {
my $component = $matchpoint->{'components'}->[$i];
$j++;
last FIELD if $j > 0 and $check_only_first_repeat;
last FIELD if $i > 0 and $j > $#keys;
- my $key = "";
- if ($field->is_control_field()) {
- if ($component->{'length'}) {
- $key = _normalize(substr($field->data(), $component->{'offset'}, $component->{'length'}))
- # FIXME normalize, substr
- } else {
- $key = _normalize($field->data());
- }
+
+ my $string;
+ if ( $field->is_control_field() ) {
+ $string = $field->data();
} else {
- foreach my $subfield ($field->subfields()) {
- if (exists $component->{'subfields'}->{$subfield->[0]}) {
- $key .= " " . $subfield->[1];
+ $string = $field->as_string(
+ join('', keys %{ $component->{ subfields } }), ' ' # ' ' as separator
+ );
+ }
+
+ if ($component->{'length'}>0) {
+ $string= substr($string, $component->{'offset'}, $component->{'length'});
+ } elsif ($component->{'offset'}) {
+ $string= substr($string, $component->{'offset'});
+ }
+
+ my $norms = $component->{'norms'};
+ my $key = $string;
+
+ foreach my $norm ( @{ $norms } ) {
+ if ( grep { $norm eq $_ } valid_normalization_routines() ) {
+ if ( $norm eq 'remove_spaces' ) {
+ $key = remove_spaces($key);
}
- }
- $key = _normalize($key);
- if ($component->{'length'}){
- if (length($key) > $component->{'length'}){
- $key = _normalize(substr($key,$component->{'offset'},$component->{'length'}));
- }
+ elsif ( $norm eq 'upper_case' ) {
+ $key = upper_case($key);
+ }
+ elsif ( $norm eq 'lower_case' ) {
+ $key = lower_case($key);
+ }
+ elsif ( $norm eq 'legacy_default' ) {
+ $key = legacy_default($key);
+ }
+ } else {
+ warn "Invalid normalization routine required ($norm)"
+ unless $norm eq 'none';
}
}
+
if ($i == 0) {
push @keys, $key if $key;
} else {
}
}
return @keys;
-
}
return $component;
}
-# FIXME - default normalizer
-sub _normalize {
- my $value = uc shift;
- $value =~ s/.;:,\]\[\)\(\/'"//g;
- $value =~ s/^\s+//;
- #$value =~ s/^\s+$//;
- $value =~ s/\s+$//;
- $value =~ s/\s+/ /g;
- #$value =~ s/[.;,\]\[\)\(\/"']//g;
- return $value;
+sub valid_normalization_routines {
+
+ return (
+ 'remove_spaces',
+ 'upper_case',
+ 'lower_case',
+ 'legacy_default'
+ );
}
1;
=head1 AUTHOR
-Koha Development Team <info@koha.org>
+Koha Development Team <http://koha-community.org/>
Galen Charlton <galen.charlton@liblime.com>
projects / koha.git / blobdiff