1 package Koha::SearchEngine::Elasticsearch::Search;
3 # Copyright 2014 Catalyst IT
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 3 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 Koha::SearchEngine::Elasticsearch::Search - search functions for Elasticsearch
27 Koha::SearchEngine::Elasticsearch::Search->new( { index => $index } );
28 my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new(
29 { index => $index } );
30 my $query = $builder->build_query('perl');
31 my $results = $searcher->search($query);
32 print "There were " . $results->total . " results.\n";
43 use base qw(Koha::SearchEngine::Elasticsearch);
45 use C4::AuthoritiesMarc;
47 use Koha::AuthorisedValues;
48 use Koha::SearchEngine::QueryBuilder;
49 use Koha::SearchEngine::Search;
50 use Koha::Exceptions::Elasticsearch;
52 use Catmandu::Store::ElasticSearch;
54 use Data::Dumper; #TODO remove
58 Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
62 my $results = $searcher->search($query, $page, $count, %options);
64 Run a search using the query. It'll return C<$count> results, starting at page
65 C<$page> (C<$page> counts from 1, anything less that, or C<undef> becomes 1.)
66 C<$count> is also the number of entries on a page.
68 C<%options> is a hash containing extra options:
74 If provided, this overrides the C<$page> value, and specifies the record as
75 an offset (i.e. the number of the record to start with), rather than a page.
84 my ($self, $query, $page, $count, %options) = @_;
86 my $params = $self->get_elasticsearch_params();
87 # 20 is the default number of results per page
88 $query->{size} = $count || 20;
89 # ES doesn't want pages, it wants a record to start from.
90 if (exists $options{offset}) {
91 $query->{from} = $options{offset};
93 $page = (!defined($page) || ($page <= 0)) ? 0 : $page - 1;
94 $query->{from} = $page * $query->{size};
96 my $elasticsearch = $self->get_elasticsearch();
98 $elasticsearch->search(
99 index => $params->{index_name},
104 die $self->process_error($@);
111 my $count = $searcher->count($query);
113 This mimics a search request, but just gets the result count instead. That's
114 faster than pulling all the data in, usually.
119 my ( $self, $query ) = @_;
121 my $params = $self->get_elasticsearch_params();
123 Catmandu::Store::ElasticSearch->new( %$params, trace_calls => 0, ) )
126 my $search = $self->store->bag->search( %$query);
127 my $count = $search->total() || 0;
133 my ( $error, $results, $facets ) = $search->search_compat(
134 $query, $simple_query, \@sort_by, \@servers,
135 $results_per_page, $offset, $expanded_facet, $branches,
139 A search interface somewhat compatible with L<C4::Search->getRecords>. Anything
140 that is returned in the query created by build_query_compat will probably
141 get ignored here, along with some other things (like C<@servers>.)
147 $self, $query, $simple_query, $sort_by,
148 $servers, $results_per_page, $offset, $expanded_facet,
149 $branches, $query_type, $scan
152 if ( !defined $offset or $offset < 0 ) {
155 $options{offset} = $offset;
156 $options{expanded_facet} = $expanded_facet;
157 my $results = $self->search($query, undef, $results_per_page, %options);
159 # Convert each result into a MARC::Record
161 # opac-search expects results to be put in the
162 # right place in the array, according to $offset
164 my $hits = $results->{'hits'};
165 foreach my $es_record (@{$hits->{'hits'}}) {
166 $records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
169 # consumers of this expect a name-spaced result, we provide the default
172 $result{biblioserver}{hits} = $hits->{'total'};
173 $result{biblioserver}{RECORDS} = \@records;
174 return (undef, \%result, $self->_convert_facets($results->{aggregations}, $expanded_facet));
177 =head2 search_auth_compat
179 my ( $results, $total ) =
180 $searcher->search_auth_compat( $query, $offset, $count, $skipmetadata, %options );
182 This has a similar calling convention to L<search>, however it returns its
183 results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>.
187 sub search_auth_compat {
188 my ($self, $query, $offset, $count, $skipmetadata, %options) = @_;
190 if ( !defined $offset or $offset <= 0 ) {
193 # Uh, authority search uses 1-based offset..
194 $options{offset} = $offset - 1;
195 my $database = Koha::Database->new();
196 my $schema = $database->schema();
197 my $res = $self->search($query, undef, $count, %options);
199 my $bib_searcher = Koha::SearchEngine::Elasticsearch::Search->new({index => 'biblios'});
201 my $hits = $res->{'hits'};
202 foreach my $es_record (@{$hits->{'hits'}}) {
203 my $record = $es_record->{'_source'};
206 # I wonder if these should be real values defined in the mapping
207 # rather than hard-coded conversions.
209 # Handle legacy nested arrays indexed with splitting enabled.
210 my $authid = $record->{ 'Local-number' }[0];
211 $authid = @$authid[0] if (ref $authid eq 'ARRAY');
213 $result{authid} = $authid;
215 if (!defined $skipmetadata || !$skipmetadata) {
216 # TODO put all this info into the record at index time so we
217 # don't have to go and sort it all out now.
218 my $authtypecode = $record->{authtype};
219 my $rs = $schema->resultset('AuthType')
220 ->search( { authtypecode => $authtypecode } );
222 # FIXME there's an assumption here that we will get a result.
223 # the original code also makes an assumption that some provided
224 # authtypecode may sometimes be used instead of the one stored
225 # with the record. It's not documented why this is the case, so
226 # it's not reproduced here yet.
227 my $authtype = $rs->single;
228 my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
229 my $marc = $self->decode_record_from_result($record);
230 my $mainentry = $marc->field($auth_tag_to_report);
233 foreach ( $mainentry->subfields() ) {
234 $reported_tag .= '$' . $_->[0] . $_->[1];
237 # Turn the resultset into a hash
238 $result{authtype} = $authtype ? $authtype->authtypetext : $authtypecode;
239 $result{reported_tag} = $reported_tag;
241 # Reimplementing BuildSummary is out of scope because it'll be hard
243 C4::AuthoritiesMarc::BuildSummary( $marc, $result{authid},
245 $result{used} = $self->count_auth_use($bib_searcher, $authid);
247 push @records, \%result;
249 return ( \@records, $hits->{'total'} );
252 =head2 count_auth_use
254 my $count = $auth_searcher->count_auth_use($bib_searcher, $authid);
256 This runs a search to determine the number of records that reference the
257 specified authid. C<$bib_searcher> must be something compatible with
258 elasticsearch, as the query is built in this function.
263 my ($self, $bib_searcher, $authid) = @_;
268 # query => { match_all => {} },
269 filter => { term => { an => $authid } }
273 $bib_searcher->count($query);
276 =head2 simple_search_compat
278 my ( $error, $marcresults, $total_hits ) =
279 $searcher->simple_search( $query, $offset, $max_results, %options );
281 This is a simpler interface to the searching, intended to be similar enough to
282 L<C4::Search::SimpleSearch>.
290 A thing to search for. It could be a simple string, or something constructed
291 with the appropriate QueryBuilder module.
295 How many results to skip from the start of the results.
297 =item C<$max_results>
299 The max number of results to return. The default is 100 (because unlimited
300 is a pretty terrible thing to do.)
304 These options are unused by Elasticsearch
314 if something went wrong, this'll contain some kind of error
317 =item C<$marcresults>
319 an arrayref of MARC::Records (note that this is different from the
320 L<C4::Search> version which will return plain XML, but too bad.)
324 the total number of results that this search could have returned.
330 sub simple_search_compat {
331 my ($self, $query, $offset, $max_results) = @_;
333 return ('No query entered', undef, undef) unless $query;
336 $offset = 0 if not defined $offset or $offset < 0;
337 $options{offset} = $offset;
338 $max_results //= 100;
340 unless (ref $query) {
341 # We'll push it through the query builder to sanitise everything.
342 my $qb = Koha::SearchEngine::QueryBuilder->new({index => $self->index});
343 (undef,$query) = $qb->build_query_compat(undef, [$query]);
345 my $results = $self->search($query, undef, $max_results, %options);
347 my $hits = $results->{'hits'};
348 foreach my $es_record (@{$hits->{'hits'}}) {
349 push @records, $self->decode_record_from_result($es_record->{'_source'});
351 return (undef, \@records, $hits->{'total'});
354 =head2 extract_biblionumber
356 my $biblionumber = $searcher->extract_biblionumber( $searchresult );
358 $searchresult comes from simple_search_compat.
360 Returns the biblionumber from the search result record.
364 sub extract_biblionumber {
365 my ( $self, $searchresultrecord ) = @_;
366 return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
369 =head2 decode_record_from_result
370 my $marc_record = $self->decode_record_from_result(@result);
372 Extracts marc data from Elasticsearch result and decodes to MARC::Record object
376 sub decode_record_from_result {
377 # Result is passed in as array, will get flattened
378 # and first element will be $result
379 my ( $self, $result ) = @_;
380 if ($result->{marc_format} eq 'base64ISO2709') {
381 return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
383 elsif ($result->{marc_format} eq 'MARCXML') {
384 return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
387 Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
391 =head2 max_result_window
393 Returns the maximum number of results that can be fetched
395 This directly requests Elasticsearch for the setting index.max_result_window (or
396 the default value for this setting in case it is not set)
400 sub max_result_window {
404 Catmandu::Store::ElasticSearch->new(%{ $self->get_elasticsearch_params })
405 ) unless $self->store;
407 my $index_name = $self->store->index_name;
408 my $settings = $self->store->es->indices->get_settings(
409 index => $index_name,
410 params => { include_defaults => 1, flat_settings => 1 },
413 my $max_result_window = $settings->{$index_name}->{settings}->{'index.max_result_window'};
414 $max_result_window //= $settings->{$index_name}->{defaults}->{'index.max_result_window'};
416 return $max_result_window;
419 =head2 _convert_facets
421 my $koha_facets = _convert_facets($es_facets, $expanded_facet);
423 Converts elasticsearch facets types to the form that Koha expects.
424 It expects the ES facet name to match the Koha type, for example C<itype>,
425 C<au>, C<su-to>, etc.
427 C<$expanded_facet> is the facet that we want to show FacetMaxCount entries for, rather
428 than just 5 like normal.
432 sub _convert_facets {
433 my ( $self, $es, $exp_facet ) = @_;
437 # These should correspond to the ES field names, as opposed to the CCL
438 # things that zebra uses.
439 # TODO let the library define the order using the interface.
440 my %type_to_label = (
441 author => { order => 1, label => 'Authors', },
442 itype => { order => 2, label => 'ItemTypes', },
443 location => { order => 3, label => 'Location', },
444 'su-geo' => { order => 4, label => 'Places', },
445 se => { order => 5, label => 'Series', },
446 subject => { order => 6, label => 'Topics', },
447 ccode => { order => 7, label => 'CollectionCodes',},
448 holdingbranch => { order => 8, label => 'HoldingLibrary' },
449 homebranch => { order => 9, label => 'HomeLibrary' }
452 # We also have some special cases, e.g. itypes that need to show the
453 # value rather than the code.
454 my @itypes = Koha::ItemTypes->search;
455 my @libraries = Koha::Libraries->search;
456 my $library_names = { map { $_->branchcode => $_->branchname } @libraries };
457 my @locations = Koha::AuthorisedValues->search( { category => 'LOC' } );
458 my $opac = C4::Context->interface eq 'opac' ;
460 itype => { map { $_->itemtype => $_->description } @itypes },
461 location => { map { $_->authorised_value => ( $opac ? ( $_->lib_opac || $_->lib ) : $_->lib ) } @locations },
462 holdingbranch => $library_names,
463 homebranch => $library_names
467 while ( my ( $type, $data ) = each %$es ) {
468 next if !exists( $type_to_label{$type} );
470 # We restrict to the most popular $limit !results
471 my $limit = ( $type eq $exp_facet ) ? C4::Context->preference('FacetMaxCount') : 5;
473 type_id => $type . '_id',
475 expandable => ( $type ne $exp_facet )
476 && ( @{ $data->{buckets} } > $limit ),
477 "type_label_$type_to_label{$type}{label}" => 1,
478 type_link_value => $type,
479 order => $type_to_label{$type}{order},
481 $limit = @{ $data->{buckets} } if ( $limit > @{ $data->{buckets} } );
482 foreach my $term ( @{ $data->{buckets} }[ 0 .. $limit - 1 ] ) {
483 my $t = $term->{key};
484 my $c = $term->{doc_count};
486 if ( exists( $special{$type} ) ) {
487 $label = $special{$type}->{$t} // $t;
492 push @{ $facet->{facets} }, {
494 facet_link_value => $t,
495 facet_title_value => $t . " ($c)",
496 facet_label_value => $label, # TODO either truncate this,
497 # or make the template do it like it should anyway
498 type_link_value => $type,
501 push @facets, $facet if exists $facet->{facets};
504 @facets = sort { $a->{order} cmp $b->{order} } @facets;