speedup facets using stored filters

[MojoFacets.git] / lib / MojoFacets / Data.pm

diff --git a/lib/MojoFacets/Data.pm b/lib/MojoFacets/Data.pm
index 333b6b2..bf150ee 100644 (file)
--- a/lib/MojoFacets/Data.pm
+++ b/lib/MojoFacets/Data.pm
@@ -115,7 +115,7 @@ sub _load_path {
                                $stats->{$n}->{numeric}++
                                        if $x =~ m/^[-+]?([0-9]*\.[0-9]+|[0-9]+)$/;
                                $stats->{$n}->{empty}++
-                                       if $x =~ m/^\s*$/;
+                                       if length $x == 0; # faster than $x =~ m/^\s*$/;
                        }
 
                }
@@ -160,12 +160,13 @@ sub load {
        $self->session( 'header' => $loaded->{$path}->{header} );
        if ( ! defined $loaded->{$path}->{columns} ) {
                $self->session( 'columns' => $loaded->{$path}->{header} );
+               $self->session( 'order'   => $loaded->{$path}->{header}->[0] );
                $self->redirect_to( '/data/columns' );
        } else {
                $self->session( 'columns' => $loaded->{$path}->{columns} );
+               $self->session( 'order'   => $loaded->{$path}->{columns}->[0] );
                $self->redirect_to( '/data/items' );
        }
-               
 }
 
 
@@ -228,7 +229,7 @@ sub _perm_array {
                        die "$name not array ",dump($session);
                }
        }
-       warn "# $name ",dump @array;
+       #warn "# $name ",dump @array;
        return @array;
 }
 
@@ -259,54 +260,64 @@ sub filter {
        my $name = $self->param('filter_name') || die "name?";
        my @vals = $self->param('filter_vals');
 
-#      warn "# filter $name vals ",dump(@vals);
-
        my $path = $self->session('path');
 
        if ( @vals ) {
-               $filters->{$name} = [ @vals ];
-               warn "# filter + $name $#vals\n";
+               $self->_filter_on_data( $name, @vals );
+       } else {
+               warn "# filter - $name\n";
+               delete $filters->{$name};
+               delete $loaded->{$path}->{filters}->{$name};
+       }
 
-               my $filter_hash;
-               $filter_hash->{$_}++ foreach @vals;
+       #warn "# filters ",dump($filters);
 
-               warn "# filter_hash ",dump( $filter_hash );
+       $self->session( 'offset' => 0 );
 
-               my $items = $self->_loaded('data')->{items};
+       $self->redirect_to('/data/items');
+}
 
-               my $include_missing = defined $filter_hash->{_missing};
-               my $filtered_items;
+sub _filter_on_data {
+       my ( $self, $name, @vals ) = @_;
 
-               foreach my $i ( 0 .. $#$items ) {
+       my $path = $self->session('path');
 
-                       if ( defined $items->[$i]->{$name} ) {
-                               foreach my $v ( @{ $items->[$i]->{$name} } ) {
-                                       if ( defined $filter_hash->{ $v } ) {
-                                               $filtered_items->{$i}++;
-                                       }
-                               }
-                       } elsif ( $include_missing ) {
-                               $filtered_items->{$i}++;
-                       }
-               }
+       if ( ! defined $loaded->{$path}->{stats}->{ $name } ) {
+               warn "filter $name not found in data set";
+               return;
+       }
 
-               warn "# filter $name ",dump($filtered_items);
+       $filters->{$name} = [ @vals ];
+       warn "_filter_on_data $name ", $#vals + 1, " values on $path\n";
 
-               $loaded->{$path}->{filters}->{$name} = $filtered_items;
+       my $filter_hash;
+       $filter_hash->{$_}++ foreach @vals;
 
-       } else {
-               warn "# filter - $name\n";
-               delete $filters->{$name};
-               delete $loaded->{$path}->{filters}->{$name};
-       }
+       #warn "# filter_hash ",dump( $filter_hash );
 
-       warn "# filters ",dump($filters);
+       my $items = $self->_loaded('data')->{items};
 
-       $self->session( 'offset' => 0 );
+       my $include_missing = defined $filter_hash->{_missing};
+       my $filtered_items;
 
-       $self->redirect_to('/data/items');
-}
+       foreach my $i ( 0 .. $#$items ) {
+
+               if ( defined $items->[$i]->{$name} ) {
+                       foreach my $v ( @{ $items->[$i]->{$name} } ) {
+                               if ( defined $filter_hash->{ $v } ) {
+                                       $filtered_items->{$i}++;
+                               }
+                       }
+               } elsif ( $include_missing ) {
+                       $filtered_items->{$i}++;
+               }
+       }
 
+       #warn "# filter $name ",dump($filtered_items);
+
+       $loaded->{$path}->{filters}->{$name} = $filtered_items;
+       warn "generated ", scalar keys %$filtered_items, " filtered items\n";
+}
 
 sub _data_items {
        my ( $self, $all ) = @_;
@@ -359,7 +370,7 @@ sub _current_filters {
                        grep { defined $filters->{ $_ } }
                        @{ $self->_loaded('header') }
                );
-       warn "# current_filters ",dump($current_filters);
+       #warn "# current_filters ",dump($current_filters);
        return $current_filters;
 }
 
@@ -415,12 +426,32 @@ sub items {
 
        my $sorted = $self->_data_sorted_by( $order );
 
+       my @filter_names;
+       if ( $filters ) {
+               foreach my $name ( keys %$filters ) {
+                       if ( ! defined $loaded->{$path}->{stats}->{ $name } ) {
+                               warn "skip filter $name not found in $path\n";
+                               next;
+                       }
+                       push @filter_names, $name;
+               }
+               warn "filter_names ",dump( @filter_names );
+               foreach my $name ( @filter_names ) {
+                       next if ref $loaded->{$path}->{filters}->{$name} eq 'ARRAY';
+                       $self->_filter_on_data( $name, @{ $filters->{$name} } );
+               }
+       }
+
        my $path_filters = $loaded->{$path}->{filters};
-       my @filter_names = keys %$path_filters;
+
+       my $sort_descending = $sort eq 'd';
 
        my @filtered;
        foreach my $i ( 0 .. $#$sorted ) {
-               my $pos = $sort eq 'd' ? $sorted->[$i] : $sorted->[ $#$sorted - $i ];
+               my $pos = $sort_descending ? $sorted->[$i] : $sorted->[ $#$sorted - $i ];
+
+               push @filtered, $pos;
+               next if $#filter_names == -1;
 
                my $skip = 0;
                foreach ( @filter_names ) {
@@ -439,7 +470,7 @@ sub items {
        }
 
 
-#      warn "# sorted ", dump $sorted;
+       warn "# sorted_items ", $#$sorted_items + 1;
 
        $self->render(
                order => $order,
@@ -481,6 +512,7 @@ sub facet {
 
        if ( my $remove = $self->param('remove') ) {
                delete $filters->{$remove};
+               delete $loaded->{$path}->{filters}->{$remove};
                $self->redirect_to( '/data/items' );
        }
 
@@ -489,13 +521,20 @@ sub facet {
 
        my $all = $self->_perm_scalar('all', 1);
 
-       foreach my $i ( $self->_data_items($all) ) {
-               if ( ! exists $i->{$name} ) {
+       my $filter = $loaded->{$path}->{filters}->{$name};
+
+       my $data = $self->_loaded('data');
+       foreach my $i ( 0 .. $#{ $data->{items} } ) {
+               if ( $filter && ! $all ) {
+                       next unless defined $loaded->{$path}->{filters}->{$name}->{$i};
+               }
+               my $item = $data->{items}->[$i];
+               if ( ! exists $item->{$name} ) {
                        $facet->{ _missing }++;
-               } elsif ( ref $i->{$name} eq 'ARRAY' ) {
-                       $facet->{$_}++ foreach @{ $i->{$name} };
+               } elsif ( ref $item->{$name} eq 'ARRAY' ) {
+                       $facet->{$_}++ foreach @{ $item->{$name} };
                } else {
-                       $facet->{ $i->{$name} }++;
+                       $facet->{ $item->{$name} }++;
                }
        }
 
@@ -520,7 +559,7 @@ sub facet {
                } elsif ( $sort =~ m/d/i ) {
                        $numeric ? $b <=> $a : lc $b cmp lc $a;
                } elsif ( $sort =~ m/c/i ) {
-                       $facet->{$b} <=> $facet->{$a};
+                       ( $facet->{$b} || -1 ) <=> ( $facet->{$a} || -1 )
                } else {
                        warn "unknown sort: $sort";
                        $a cmp $b;