Bug 17255 - Upgrade Elastic Search code to work with version 5.1

Builds on top of commit:
    Bug 17255 - Upgrade Elastic Search code to work with version 2.4+ - rebased wip

-Fix data type 'string' to 'keyword' and 'text'
-index: not_analyzed deprecated, replaced with type: keyword which is equivalent
-store: yes was deprecated, use store: true

TODO: Installer bindings to both the debian package install and the raw developer install.
A taster in Buug 17851

ZE TEST PLAN

0. Remove existing ES and reinstall ES 5.1
   apt-get purge elasticsearch
   Follow instruction here:
   https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html

1. Reset Elasticsearch index since facets are hard coded to dynamic search_marc_mappings.
1a. perl -e 'use Koha::SearchEngine::Elasticsearch; Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings();'
1b. If you get trouble, simply DELETE FROM [search_fields|search_marc_to_fields|search_marc_mapping];
    and retry 1a.
1c. Destroy elasticsearch index
    curl -XDELETE localhost:9200/koha_biblios
    so it can be recreated

2. Recreate the index:
   perl misc/search_tools/rebuild_elastic_search.pl
2a. Add something to index if your koha.biblio-table is empty

3. Fetch all indexed records and the facet for subject__facet

curl -XGET localhost:9200/koha_biblios/data/_search?pretty -d '{
  "aggregations": {
    "my_agg": {
      "terms": {
        "field": "subject__facet"
      }
    }
  }
}'

4. Run the included tests:
   perl t/db_dependent/Koha_Elasticsearch_Indexer.t
   perl t/db_dependent/Koha_Elasticsearch.t
   perl t/db_dependent/Koha_SearchEngine_Elasticsearch_Search.t

Have fun with your new ES 5.1 cluster!

Signed-off-by: Nick Clemens <nick@bywatersolutions.com>
Signed-off-by: Tomas Cohen Arazi <tomascohen@theke.io>
Signed-off-by: Kyle M Hall <kyle@bywatersolutions.com>

diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm
index f8b904c..03cc5c3 100644 (file)
--- a/Koha/SearchEngine/Elasticsearch.pm
+++ b/Koha/SearchEngine/Elasticsearch.pm
@@ -148,10 +148,6 @@ sub get_elasticsearch_settings {
                         tokenizer => 'standard',
                         filter    => ['lowercase'],
                     },
-                    default => {
-                        tokenizer => 'keyword',
-                        filter    => ['lowercase'],
-                    },
                 },
             }
         }
@@ -176,9 +172,9 @@ sub get_elasticsearch_mappings {
         data => {
             properties => {
                 record => {
-                    store          => "yes",
+                    store          => "true",
                     include_in_all => JSON::false,
-                    type           => "string",
+                    type           => "text",
                 },
             }
         }
@@ -189,7 +185,6 @@ sub get_elasticsearch_mappings {
         sub {
             my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
             return if $marc_type ne $marcflavour;
-
             # TODO if this gets any sort of complexity to it, it should
             # be broken out into its own function.
 
@@ -198,7 +193,7 @@ sub get_elasticsearch_mappings {
             my $es_type =
               $type eq 'boolean'
               ? 'boolean'
-              : 'string';
+              : 'text';
 
             if ($es_type eq 'boolean') {
                 $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_boolean( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
@@ -209,8 +204,7 @@ sub get_elasticsearch_mappings {
 
             if ($facet) {
                 $mappings->{data}{properties}{ $name . '__facet' } = {
-                    type  => "string",
-                    index => "not_analyzed",
+                    type  => "keyword",
                 };
             }
             if ($suggestible) {
@@ -226,13 +220,13 @@ sub get_elasticsearch_mappings {
                 $mappings->{data}{properties}{ $name . '__sort' } = {
                     search_analyzer => "analyser_phrase",
                     analyzer  => "analyser_phrase",
-                    type            => "string",
+                    type            => "text",
                     include_in_all  => JSON::false,
                     fields          => {
                         phrase => {
                             search_analyzer => "analyser_phrase",
                             analyzer  => "analyser_phrase",
-                            type            => "string",
+                            type            => "text",
                         },
                     },
                 };
@@ -272,11 +266,10 @@ sub _elasticsearch_mapping_for_default {
             phrase => {
                 search_analyzer => "analyser_phrase",
                 analyzer        => "analyser_phrase",
-                type            => "string",
+                type            => "text",
             },
             raw => {
-                type    => "string",
-                index   => "not_analyzed",
+                type    => "keyword",
             }
         },
     };

diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm
index 172580a..e5babee 100644 (file)
--- a/Koha/SearchEngine/Elasticsearch/Indexer.pm
+++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm
@@ -77,7 +77,8 @@ sub update_index {
             )
         );
     }
-    my $array_ref = $from->to_array;
+
+    #print Data::Dumper::Dumper( $from->to_array );
     $self->store->bag->add_many($from);
     $self->store->bag->commit;
     return 1;

diff --git a/Koha/SearchEngine/Elasticsearch/Search.pm b/Koha/SearchEngine/Elasticsearch/Search.pm
index 6e3ca3b..d3cd294 100644 (file)
--- a/Koha/SearchEngine/Elasticsearch/Search.pm
+++ b/Koha/SearchEngine/Elasticsearch/Search.pm
@@ -257,8 +257,8 @@ sub count_auth_use {
 
     my $query = {
         query => {
-            filtered => {
-                query  => { match_all => {} },
+            bool => {
+#                query  => { match_all => {} },
                 filter => { term      => { an => $authid } }
             }
         }

diff --git a/t/db_dependent/Koha_Elasticsearch. b/t/db_dependent/Koha_Elasticsearch.
deleted file mode 100644 (file)
index bad08d2..0000000
--- a/t/db_dependent/Koha_Elasticsearch.
+++ /dev/null
@@ -1,23 +0,0 @@
-# Copyright 2015 Catalyst IT
-#
-# This file is part of Koha.
-#
-# Koha is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# Koha is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Koha; if not, see <http://www.gnu.org/licenses>
-
-use strict;
-use warnings;
-
-use Test::More tests => 1;                      # last test to print
-
-use_ok('Koha::Search::Engine::Elasticsearch');

diff --git a/t/db_dependent/Koha_Elasticsearch.t b/t/db_dependent/Koha_Elasticsearch.t
new file mode 100644 (file)
index 0000000..eb24cbb
--- /dev/null
+++ b/t/db_dependent/Koha_Elasticsearch.t
@@ -0,0 +1,23 @@
+# Copyright 2015 Catalyst IT
+#
+# This file is part of Koha.
+#
+# Koha is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# Koha is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Koha; if not, see <http://www.gnu.org/licenses>
+
+use strict;
+use warnings;
+
+use Test::More tests => 1;                      # last test to print
+
+use_ok('Koha::SearchEngine::Elasticsearch');