X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=lib%2FBackupPC%2FSearch%2FKinoSearch.pm;h=e6f7c6a3b3e42a91d18b8118c01cae5cf4376717;hp=f6b3f915f13b8b3b8f4ca947e80ddeeae9ea10be;hb=ae81c69bf053eed787bac83c224225446ac07449;hpb=0a2303cdbfbc9ab4b83edf46635a73fb7323b2f2 diff --git a/lib/BackupPC/Search/KinoSearch.pm b/lib/BackupPC/Search/KinoSearch.pm index f6b3f91..e6f7c6a 100644 --- a/lib/BackupPC/Search/KinoSearch.pm +++ b/lib/BackupPC/Search/KinoSearch.pm @@ -15,6 +15,20 @@ sub new { my $class = shift @_; my %Conf = @_; + my $index_path = $Conf{KinoPath} || die "no KinoPath"; + + my $self = bless { + index => $index_path, + first_time_indexing => ! -d $index_path, + }, $class; + warn "# ",dump($self); + return $self; +} + +sub indexer { + my $self = shift; + return $self->{_indexer} if defined $self->{_indexer}; + my $schema = KinoSearch::Plan::Schema->new; @@ -29,51 +43,84 @@ sub new { ); my $blob_type = KinoSearch::Plan::BlobType->new( stored => 1 ); my $string_type = KinoSearch::Plan::StringType->new; # non-tokenized - my $num_type = KinoSearch::Plan::Int64Type->new; + my $num_type = KinoSearch::Plan::Int64Type->new( sortable => 1 ); + my $sort_type = KinoSearch::Plan::StringType->new( sortable => 1 ); # non-tokenized - $schema->spec_field( name => $_, type => $string_type ) foreach ( qw/ - backup_date +=for numeric-no-padding + fid + shareid + type backupnum +=cut + $self->{numeric_padding} = [ qw/ + backup_date date + size + / ]; + + # numeric + $schema->spec_field( name => $_, type => $string_type ) foreach ( qw/ + backup_date fid shareid - size type / ); + # non-tokenized strings $schema->spec_field( name => $_, type => $string_type ) foreach ( qw/ - _uri _file_path_split filepath hname sname + _uri + hname /); -# $schema->spec_field( name => '_doc', type => $blob_type ); + # sortable + $schema->spec_field( name => $_, type => $sort_type ) foreach (qw/ + sname + filepath + /); - my $index_path = $Conf{KinoPath} || die "no KinoPath"; + # sortable numeric + $schema->spec_field( name => $_, type => $sort_type ) foreach (qw/ + backupnum + date + size + /); + + # tokenized magic columns for infix search + $schema->spec_field( name => '_file_path_split', type => $ft_type ); + +# $schema->spec_field( name => '_doc', type => $blob_type ); my $indexer = KinoSearch::Index::Indexer->new( schema => $schema, - index => $index_path, + index => $self->{index}, create => 1, ); - warn "# using $index_path"; + warn "# created indexer"; - $indexer->commit; # make sure that index exists + return $self->{_indexer} = $indexer; - my $self = bless { - indexer => $indexer, - searcher => KinoSearch::Search::IndexSearcher->new( - index => $index_path, - ), +}; - }, $class; - return $self; +our $searcher; +sub searcher { + my $self = shift; + return $self->{_searcher} if $self->{_searcher}; + $self->{_searcher} = + KinoSearch::Search::IndexSearcher->new( index => $self->{index} ) } sub exists { my ($self,$row) = @_; + return 0 if $self->{first_time_indexing}; + my $uri = $row->{hname} . ':' . $row->{sname} . '#' . $row->{backupnum} . ' ' . $row->{filepath}; - my $hits = $self->{searcher}->hits( query => "_uri:$uri" ); + my $hits = $self->searcher->hits( query => "_uri:$uri" ); + + + $self->{stat}->{exists}->{ $hits->total_hits }++; + return $hits->total_hits; } @@ -85,16 +132,101 @@ sub add_doc { $path =~ s/(.)/$1 /g; # XXX our tokenize $row->{_file_path_split} = $path; + $self->{stats}->{add_doc}++; + + foreach my $col ( @{ $self->{numeric_padding} } ) { + $row->{$col} = sprintf "%011d", $row->{$col}; + } + warn "XXX ",dump($row) if $ENV{DEBUG}; - $self->{indexer}->add_doc( $row ); + $self->indexer->add_doc( $row ); } sub commit { my $self = shift; - $self->{indexer}->commit; - warn "# commit index"; + $self->indexer->commit; + warn "# commit index ", dump($self->{stats}); +} + +sub _field_lower_upper_term { + my ( $field, $l, $u ) = @_; + my $range; + if ( $l ) { + $range->{lower_term} = $l; + $range->{include_lower} = 1; + } + if ( $u ) { + $range->{upper_term} = $u; + $range->{include_upper} = 1; + } + $range->{field} = $field if $range; + warn "# $field $l - $u ",dump($range); + return $range; +} + +sub search { + my ( $self, $offset, $on_page, $sort, $q, $shareid, $backup_from, $backup_to, $files_from, $files_to ) = @_; + + warn "# search $offset/$on_page [$q] shareid: $shareid backup: $backup_from - $backup_to files: $files_from - $files_to"; + + my $sort_field = (split(/_/,$sort,2))[0]; + + my $rules = [ KinoSearch::Search::SortRule->new( type => 'score' ) ]; + $rules->[0] = KinoSearch::Search::SortRule->new( field => $sort_field, reverse => $sort =~ m/_a$/ ? 0 : 1 ) if $sort_field; + + my $sort_spec = KinoSearch::Search::SortSpec->new( rules => $rules ); + + my $split = $q; + $split =~ s/(.)/$1 /g; # _file_path_split + my $split_query = KinoSearch::Search::TermQuery->new( field => '_file_path_split', term => $split ); +#warn "XXX ",dump($split_query); + + + my $query_parser = KinoSearch::Search::QueryParser->new( + schema => $self->searcher->get_schema, + fields => ['_file_path_split'], + ); + my $query = $query_parser->parse( '"' . $split . '"' ); + + my @and_query; + + if ( $shareid ) { + push @and_query, KinoSearch::Search::TermQuery->new( field => 'shareid', term => $shareid ); + } + + if ( my $range = _field_lower_upper_term( 'backup_date', $backup_from, $backup_to ) ) { + push @and_query, KinoSearch::Search::RangeQuery->new( %$range ); + } + if ( my $range = _field_lower_upper_term( 'date', $files_from, $files_to ) ) { + push @and_query, KinoSearch::Search::RangeQuery->new( %$range ); + } + + if ( @and_query ) { + push @and_query, $query; + $query = KinoSearch::Search::ANDQuery->new( children => [ @and_query ] ); + } + + my $hits = $self->searcher->hits( + query => m/:/ ? $q : $query, + offset => $offset, + num_wanted => $on_page, + sort_spec => $sort_spec, + ); + + + warn "# ", $hits->total_hits, " hits for $q\n"; + + return (0,[]) if $hits->total_hits == 0; + + my $results; + while ( my $hit = $hits->next ) { + warn "## hit = ",dump($hit) if $ENV{DEBUG}; + push @$results, $hit; + } + + return ( $hits->total_hits, $results ); } 1;