1 package BackupPC::Search::KinoSearch;
5 use KinoSearch::Index::Indexer;
6 use KinoSearch::Plan::Schema;
7 use KinoSearch::Analysis::PolyAnalyzer;
8 use KinoSearch::Plan::FullTextType;
9 use KinoSearch::Search::IndexSearcher;
10 use Data::Dump qw(dump);
12 # my $tokenizer = KinoSearch::Analysis::Tokenizer->new( pattern => '\\w' );
14 # numeric_padding values are used in range search, and muse be sortable
20 my $index_path = $Conf{KinoPath} || die "no KinoPath";
24 first_time_indexing => ! -d $index_path,
25 numeric_padding => [ qw(
31 warn "# ",dump($self);
37 return $self->{_indexer} if defined $self->{_indexer};
39 my $schema = KinoSearch::Plan::Schema->new;
42 my $case_folder = KinoSearch::Analysis::CaseFolder->new;
43 my $tokenizer = KinoSearch::Analysis::Tokenizer->new;
44 my $polyanalyzer = KinoSearch::Analysis::PolyAnalyzer->new(
45 analyzers => [ $case_folder, $tokenizer ],
48 my $ft_type = KinoSearch::Plan::FullTextType->new(
49 analyzer => $polyanalyzer,
51 my $blob_type = KinoSearch::Plan::BlobType->new( stored => 1 );
52 my $string_type = KinoSearch::Plan::StringType->new; # non-tokenized
53 my $num_type = KinoSearch::Plan::Int64Type->new( sortable => 1 );
54 my $sort_type = KinoSearch::Plan::StringType->new( sortable => 1 ); # non-tokenized
57 $schema->spec_field( name => $_, type => $string_type ) foreach ( qw/
63 # non-tokenized strings
64 $schema->spec_field( name => $_, type => $string_type ) foreach ( qw/
70 $schema->spec_field( name => $_, type => $sort_type ) foreach (qw/
76 $schema->spec_field( name => $_, type => $sort_type ) foreach (qw/
83 # tokenized magic columns for infix search
84 $schema->spec_field( name => '_file_path_split', type => $ft_type );
86 # $schema->spec_field( name => '_doc', type => $blob_type );
88 my $indexer = KinoSearch::Index::Indexer->new(
90 index => $self->{index},
94 warn "# created indexer";
96 return $self->{_indexer} = $indexer;
103 return $self->{_searcher} if $self->{_searcher};
105 KinoSearch::Search::IndexSearcher->new( index => $self->{index} )
109 my ($self,$row) = @_;
111 return 0 if $self->{first_time_indexing};
113 my $uri = $row->{hname} . ':' . $row->{sname} . '#' . $row->{backupnum} . ' ' . $row->{filepath};
114 my $hits = $self->searcher->hits( query => "_uri:$uri" );
117 $self->{stat}->{exists}->{ $hits->total_hits }++;
119 return $hits->total_hits;
122 sub _numeric_padding { sprintf "%010d", $_[0] } # pad up to 32bit number (timestamp)
125 my ($self,$row) = @_;
127 $row->{_uri} = $row->{hname} . ':' . $row->{sname} . '#' . $row->{backupnum} . ' ' . $row->{filepath};
128 my $path = $row->{filepath};
129 $path =~ s/(.)/$1 /g; # XXX our tokenize
130 $row->{_file_path_split} = $path;
132 $self->{stats}->{add_doc}++;
134 foreach my $col ( @{ $self->{numeric_padding} } ) {
135 $row->{$col} = _numeric_padding $row->{$col};
138 warn "XXX ",dump($row) if $ENV{DEBUG};
140 $self->indexer->add_doc( $row );
146 $self->indexer->commit;
147 warn "# commit index ", dump($self->{stats});
150 sub _field_lower_upper_term {
151 my ( $self, $field, $l, $u ) = @_;
152 my $numeric_padding = grep { /^$field$/ } @{ $self->{numeric_padding} };
155 $range->{lower_term} = $numeric_padding ? _numeric_padding $l : $l;
156 $range->{include_lower} = 1;
159 $range->{upper_term} = $numeric_padding ? _numeric_padding $u : $u;
160 $range->{include_upper} = 1;
163 $range->{field} = $field;
165 warn "# $field $l - $u numeric_padding:$numeric_padding ",dump($range);
171 my ( $self, $offset, $on_page, $sort, $q, $shareid, $backup_from, $backup_to, $files_from, $files_to ) = @_;
173 warn "# search $offset/$on_page [$q] shareid: $shareid backup: $backup_from - $backup_to files: $files_from - $files_to";
175 my $sort_field = (split(/_/,$sort,2))[0];
177 my $rules = [ KinoSearch::Search::SortRule->new( type => 'score' ) ];
178 $rules->[0] = KinoSearch::Search::SortRule->new( field => $sort_field, reverse => $sort =~ m/_a$/ ? 0 : 1 ) if $sort_field;
180 my $sort_spec = KinoSearch::Search::SortSpec->new( rules => $rules );
183 $split =~ s/(.)/$1 /g; # _file_path_split
184 my $split_query = KinoSearch::Search::TermQuery->new( field => '_file_path_split', term => $split );
185 #warn "XXX ",dump($split_query);
188 my $query_parser = KinoSearch::Search::QueryParser->new(
189 schema => $self->searcher->get_schema,
190 fields => ['_file_path_split'],
192 my $query = $query_parser->parse( '"' . $split . '"' );
197 push @and_query, KinoSearch::Search::TermQuery->new( field => 'shareid', term => $shareid );
200 if ( my $range = $self->_field_lower_upper_term( 'backup_date', $backup_from, $backup_to ) ) {
201 push @and_query, KinoSearch::Search::RangeQuery->new( %$range );
203 if ( my $range = $self->_field_lower_upper_term( 'date', $files_from, $files_to ) ) {
204 push @and_query, KinoSearch::Search::RangeQuery->new( %$range );
208 push @and_query, $query;
209 $query = KinoSearch::Search::ANDQuery->new( children => [ @and_query ] );
212 my $hits = $self->searcher->hits(
213 query => m/:/ ? $q : $query,
215 num_wanted => $on_page,
216 sort_spec => $sort_spec,
220 warn "# ", $hits->total_hits, " hits for $q\n";
222 return (0,[]) if $hits->total_hits == 0;
225 while ( my $hit = $hits->next ) {
226 warn "## hit = ",dump($hit) if $ENV{DEBUG};
227 push @$results, $hit;
230 return ( $hits->total_hits, $results );