X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=lib%2FMojoFacets%2FData.pm;h=e69c16e5fbac008e77afa35f88b6fbe7a2a862c5;hb=2c8d9972340cb72f8eb5c0eb1385eb32ae657144;hp=6db5aa26e2b55d065a90750610fa26cc6ba7535d;hpb=d249d43cd05e87ffb3d28eadf3cbfcb44b9ff6b7;p=MojoFacets.git diff --git a/lib/MojoFacets/Data.pm b/lib/MojoFacets/Data.pm index 6db5aa2..e69c16e 100644 --- a/lib/MojoFacets/Data.pm +++ b/lib/MojoFacets/Data.pm @@ -9,6 +9,11 @@ use Data::Dump qw(dump); use File::Slurp; use JSON; use Encode; +use locale; +use File::Find; + +our $loaded; +our $filters; sub index { my $self = shift; @@ -16,51 +21,112 @@ sub index { my $path = $self->app->home->rel_dir('data'); die "no data dir $path" unless -d $path; - opendir(my $dir, $path) || die $!; - my @files = - grep { -f "$path/$_" && $_ =~ m/\.js(on)?$/ } - readdir $dir; - close($dir); + my @files; + find( sub { + my $file = $File::Find::name; + if ( -f $file && $file =~ m/\.(js(on)?|txt)$/ ) { + $file =~ s/$path\/*//; + push @files, $file; + } else { + warn "IGNORE: $file\n"; + } + }, $path); - $self->render( files => [ @files ] ); -} + @files = sort { lc $a cmp lc $b } @files; + my $size; + $size->{$_} = -s "$path/$_" foreach @files; -our $data; -our $stats; + $self->render( + files => [ @files ], + size => $size, + loaded => $loaded, + filters => $filters, + ); +} -sub load { - my $self = shift; +sub _load_path { + my ( $self, $path ) = @_; - my $path = $self->app->home->rel_file( 'data/' . $self->param('path') ); - die "$path $!" unless -r $path; + my $full_path = $self->app->home->rel_file( 'data/' . $path ); + die "$full_path $!" unless -r $full_path; - $self->session('path' => $self->param('path')); + if ( defined $loaded->{$path}->{data} ) { + my $mtime = (stat($full_path))[9]; + return if $loaded->{$path}->{mtime} == $mtime; + warn "reload $full_path, modified ", time() - $mtime, " seconds ago\n"; + } # we could use Mojo::JSON here, but it's too slow # $data = from_json read_file $path; - $data = read_file $path; - Encode::_utf8_on($data); - warn "# json snippet: ", substr($data,0,200); - $data = from_json $data; + my $data = read_file $full_path; + warn "# data snippet: ", substr($data,0,200); + my @header; + if ( $path =~ m/\.js/ ) { + Encode::_utf8_on($data); + $data = from_json $data; + } elsif ( $path =~ m/\.txt/ ) { + my @lines = split(/\r?\n/, $data); + $data = { items => [] }; + + my $header_line = shift @lines; + my $multiline = $header_line =~ s/\^//g; + @header = split(/\|/, $header_line ); + warn "# header ", dump( @header ); + while ( my $line = shift @lines ) { + $line =~ s/\^//g; + chomp $line; + my @v = split(/\|/, $line); + while ( @lines && $#v < $#header ) { + $line = $lines[0]; + $line =~ s/\^//g; + chomp $line; + my @more_v = split(/\|/, $line); + if ( $#v + $#more_v > $#header ) { + warn "short line: ",dump( @v ); + last; + } + shift @lines; + $v[ $#v ] .= shift @more_v if @more_v; + push @v, @more_v if @more_v; + + if ( $#v > $#header ) { + die "# splice $#header ", dump( @v ); + @v = splice @v, 0, $#header; + } + } + my $item; + foreach my $i ( 0 .. $#v ) { + my $v = $v[$i]; + # unpack numeric values separated by commas + my $a = $v =~ m/\d+\s*,\s*\d+/ ? [ split(/\,\s*/,$v) ] : [ $v ]; + $item->{ $header[$i] || "f_$i" } = $a; + } + push @{ $data->{items} }, $item; + } + } else { + warn "file format unknown $path"; + } - $stats = {}; + my $stats; foreach my $e ( @{ $data->{items} } ) { foreach my $n ( keys %$e ) { $stats->{$n}->{count}++; + my @v; if ( ref $e->{$n} eq 'ARRAY' ) { - $stats->{$n}->{array} += $#{ $e->{$n} } + 1; - - foreach my $x ( @{$e->{$n}} ) { - $stats->{$n}->{numeric}++ - if $x =~ m/^[-+]?([0-9]*\.[0-9]+|[0-9]+)$/; - } - + @v = @{ $e->{$n} }; } else { + @v = ( $e->{$n} ); + } + + foreach my $x ( @v ) { $stats->{$n}->{numeric}++ - if $e->{$n} =~ m/^[-+]?([0-9]*\.[0-9]+|[0-9]+)$/; + if $x =~ m/^[-+]?([0-9]*\.[0-9]+|[0-9]+)$/; + $stats->{$n}->{empty}++ + if length $x == 0; # faster than $x =~ m/^\s*$/; } + } } @@ -70,25 +136,95 @@ sub load { if $stats->{$n}->{array} == $stats->{$n}->{count}; } + if ( ! @header ) { + if ( defined $data->{header} ) { + if ( ref $data->{header} eq 'ARRAY' ) { + @header = @{ $data->{header} }; + } else { + warn "header not array ", dump( $data->{header} ); + } + } + } + + @header = + sort { $stats->{$b}->{count} <=> $stats->{$a}->{count} } + grep { defined $stats->{$_}->{count} } keys %$stats + unless @header; + warn dump($stats); - $self->redirect_to( '/data/columns' ); + $loaded->{ $path } = { + header => [ @header ], + stats => $stats, + full_path => $full_path, + size => -s $full_path, + mtime => (stat($full_path))[9], + data => $data, + }; + +} + + +sub load { + my $self = shift; + + my @paths = $self->param('paths'); + warn "# paths ", dump @paths; + $self->_load_path( $_ ) foreach @paths; + + my $path = $self->param('path') || $self->redirect_to( '/data/index' ); + warn "# path $path\n"; + $self->session('path' => $path); + $self->_load_path( $path ); + + $self->session( 'header' => $loaded->{$path}->{header} ); + if ( ! defined $loaded->{$path}->{columns} ) { + $self->session( 'columns' => $loaded->{$path}->{header} ); + $self->session( 'order' => $loaded->{$path}->{header}->[0] ); + $self->redirect_to( '/data/columns' ); + } else { + $self->session( 'columns' => $loaded->{$path}->{columns} ); + $self->session( 'order' => $loaded->{$path}->{columns}->[0] ); + $self->redirect_to( '/data/items' ); + } +} + + +sub _loaded { + my ( $self, $name ) = @_; + my $path = $self->session('path'); + die "$path $name doesn't exist in loaded ",dump( $loaded ) + unless defined $loaded->{$path}->{$name}; + return $loaded->{$path}->{$name}; +} + + +sub _checked { + my $self = shift; + my $checked; + $checked->{$_}++ foreach @_; +# warn "# _checked ",dump($checked); + return $checked; } sub columns { my $self = shift; + if ( $self->param('columns') ) { + $self->_perm_array('columns'); + $self->redirect_to('/data/items'); + } + + my $stats = $self->_loaded( 'stats' ); # || $self->redirect_to( '/data/index' ); my @columns; - @columns = @{ $self->session('columns') } if $self->session('columns'); + @columns = grep { defined $stats->{$_}->{count} } @{ $self->session('columns') } if $self->session('columns'); - foreach my $c ( sort { $stats->{$b}->{count} <=> $stats->{$a}->{count} } keys %$stats ) { + foreach my $c ( @{ $self->session( 'header' ) } ) { push @columns, $c unless grep { /^\Q$c\E$/ } @columns; } - $self->redirect_to( '/data/index' ) unless @columns; - $self->render( message => 'Select columns to display', stats => $stats, @@ -101,9 +237,11 @@ sub _perm_array { my ($self,$name) = @_; my @array = $self->param($name); + my $path = $self->session('path'); if ( @array ) { $self->session($name => [ @array ]); + $loaded->{$path}->{$name} = [ @array ]; } elsif ( my $session = $self->session($name) ) { if ( ref $session eq 'ARRAY' ) { @array = @$session; @@ -111,7 +249,7 @@ sub _perm_array { die "$name not array ",dump($session); } } - warn "# $name ",dump @array; + #warn "# $name ",dump @array; return @array; } @@ -132,7 +270,7 @@ sub _perm_scalar { $self->session($name => $scalar); } - warn "# $name ",dump $scalar; + warn "# _perm_scalar $name ",dump $scalar; return $scalar; } @@ -142,147 +280,365 @@ sub filter { my $name = $self->param('filter_name') || die "name?"; my @vals = $self->param('filter_vals'); - warn "# filter $name vals ",dump(@vals); + $self->_remove_filter( $name ); + $self->_filter_on_data( $name, @vals ) if @vals; - my $filters = $self->session('filters'); - if ( @vals ) { - $filters->{$name} = [ @vals ]; - } else { - delete $filters->{$name}; + $self->session( 'offset' => 0 ); + + $self->redirect_to('/data/items'); +} + +sub _filter_on_data { + my ( $self, $name, @vals ) = @_; + + my $path = $self->session('path'); + + if ( ! defined $loaded->{$path}->{stats}->{ $name } ) { + warn "filter $name not found in data set"; + return; } - $self->session( 'filters' => $filters ); - warn "# filters ",dump($self->session('filters')); + $filters->{$name} = [ @vals ]; + warn "_filter_on_data $name ", $#vals + 1, " values on $path\n"; - $self->session( 'offset' => 0 ); + my $filter_hash; + $filter_hash->{$_}++ foreach @vals; - $self->redirect_to('/data/table'); -} + #warn "# filter_hash ",dump( $filter_hash ); -sub _filter_item { - my ( $self, $filters, $i ) = @_; - my $pass = 1; - foreach my $n ( keys %$filters ) { - # filter must exists in element - if ( ! defined $i->{$n} ) { - $pass = 0; - last; - } - # and match any of values in element - my $have_values = 0; - foreach my $v ( @{ $i->{$n} } ) { # FIXME not array? - $have_values ||= 1 if grep { m/^\Q$v\E$/ } @{ $filters->{$n} }; - } - if ( ! $have_values ) { - $pass = 0; - last; + my $items = $self->_loaded('data')->{items}; + + my $include_missing = defined $filter_hash->{_missing}; + my $filtered_items; + + foreach my $i ( 0 .. $#$items ) { + + if ( defined $items->[$i]->{$name} ) { + foreach my $v ( @{ $items->[$i]->{$name} } ) { + if ( defined $filter_hash->{ $v } ) { + $filtered_items->{$i}++; + } + } + } elsif ( $include_missing ) { + $filtered_items->{$i}++; } } - return $pass; + + #warn "# filter $name ",dump($filtered_items); + + $loaded->{$path}->{filters}->{$name} = $filtered_items; + warn "filter $name with ", scalar keys %$filtered_items, " items created\n"; } sub _data_items { + my ( $self, $all ) = @_; + my $data = $self->_loaded( 'data' ); + + return @{ $data->{items} } if $all == 1; + + my $filters = $self->_current_filters; + my $filter_value; + foreach my $f ( keys %$filters ) { + foreach my $n ( @{ $filters->{$f} } ) { + $filter_value->{$f}->{$n} = 1; + } + } + my @items = @{ $data->{items} }; + @items = grep { + my $i = $_; + my $pass = 1; + foreach my $n ( keys %$filter_value ) { + if ( ! exists $i->{$n} ) { + if ( defined $filter_value->{$n}->{_missing} ) { + $pass = 1; + next; + } else { + $pass = 0; + last; + } + } + # and match any of values in element + my $have_values = 0; + foreach my $v ( @{ $i->{$n} } ) { # FIXME not array? + $have_values ||= 1 if defined $filter_value->{$n}->{$v}; + } + if ( ! $have_values ) { + $pass = 0; + last; + } + } + $pass; + } @items if $filter_value; + return @items; +} + + +sub _current_filters { my $self = shift; - my $filters = $self->session('filters'); - grep { - $filters ? $self->_filter_item( $filters, $_ ) : 1; - } @{ $data->{items} }; + my $current_filters; + $current_filters->{ $_ } = $filters->{ $_ } + foreach ( + grep { defined $filters->{ $_ } } + @{ $self->_loaded('header') } + ); + #warn "# current_filters ",dump($current_filters); + return $current_filters; } -sub table { - my $self = shift; +sub _data_sorted_by { + my ( $self, $order ) = @_; + + my $path = $self->session('path'); + + warn "_data_sorted_by $order from $path"; + + if ( defined $loaded->{$path}->{sorted}->{$order} ) { + return $loaded->{$path}->{sorted}->{$order}; + } + + my $data = $self->_loaded( 'data' ); + my $numeric = $self->_is_numeric($order); + my $missing = $numeric ? 0 : ''; + no warnings qw(numeric); + my $nr = 0; + my @sorted = map { + $_->[0] + } sort { + if ( $numeric ) { + $a->[1] <=> $b->[1] + } else { + $a->[1] cmp $b->[1] + } + } map { + [ $nr++, exists $_->{$order} ? join('', @{$_->{$order}}) : $missing ] + } @{ $data->{items} } + ; + + warn "sorted: $order numeric: $numeric items: ", $#sorted + 1, "\n"; + #warn "# sorted ",dump( @sorted ); + + $loaded->{$path}->{sorted}->{$order} = [ @sorted ]; +} - $self->redirect_to('/data/index') unless $data->{items}; + +sub items { + my $self = shift; + + my $path = $self->session('path'); + $self->redirect_to('/data/index') unless defined $loaded->{ $path }; my @columns = $self->_perm_array('columns'); $self->redirect_to('/data/columns') unless @columns; my $order = $self->_perm_scalar('order', $columns[0]); + my $sort = $self->_perm_scalar('sort', 'a'); my $offset = $self->_perm_scalar('offset', 0); my $limit = $self->_perm_scalar('limit', 20); + $self->_perm_scalar('show', 'table'); + + # fix offset when changing limit + $offset = int( $offset / $limit ) * $limit; + + my $sorted = $self->_data_sorted_by( $order ); + + my @filter_names; + if ( $filters ) { + foreach my $name ( keys %$filters ) { + if ( ! defined $loaded->{$path}->{stats}->{ $name } ) { + warn "skip filter $name not found in $path\n"; + next; + } + push @filter_names, $name; + } + warn "filter_names ",dump( @filter_names ); + foreach my $name ( @filter_names ) { + next if ref $loaded->{$path}->{filters}->{$name} eq 'ARRAY'; + $self->_filter_on_data( $name, @{ $filters->{$name} } ); + } + } - my @sorted = sort { - ( $a->{$order} || '' ) cmp ( $b->{$order} || '' ) # FIXME - multi-level sort - } $self->_data_items; + my $all_filters = join(' ',sort @filter_names,'order:',$order); -# warn "# sorted ", dump @sorted; +# warn "# all_filters $all_filters ", dump( $loaded->{$path}->{filtered}->{$all_filters} ); - my $rows = $#sorted + 1; + if ( ! defined $loaded->{$path}->{filtered}->{$all_filters} ) { - warn "$rows $offset $limit"; + my $path_filters = $loaded->{$path}->{filters}; + + warn "create combined filter for $all_filters\n"; + + my @filtered; + foreach my $i ( 0 .. $#$sorted ) { + my $pos = $sorted->[$i]; + + if ( $#filter_names == -1 ) { + push @filtered, $pos; + next; + } + + my $skip = 0; + foreach ( @filter_names ) { + $skip ||= 1 if ! defined $path_filters->{$_}->{$pos}; + } + next if $skip; + + push @filtered, $pos; + } + + $loaded->{$path}->{filtered}->{$all_filters} = [ @filtered ]; + } + + my $filtered = $loaded->{$path}->{filtered}->{$all_filters} + if defined $loaded->{$path}->{filtered}->{$all_filters}; + + warn "all_filters $all_filters produced ", $#$filtered + 1, " items\n" if $filtered; + + my $sorted_items; + my $data = $self->_loaded('data'); + my $from_end = $sort eq 'd' ? $#$filtered : 0; + foreach ( 0 .. $limit ) { + my $i = $_ + $offset; + last unless defined $filtered->[$i]; + $i = $from_end - $i if $from_end; + push @$sorted_items, $data->{items}->[ $filtered->[$i] ]; + } + + warn "# sorted_items ", $#$sorted_items + 1, " offset $offset limit $limit order $sort"; $self->render( order => $order, offset => $offset, limit => $limit, - sorted => [ splice @sorted, $offset, $limit ], + sorted => $sorted_items, columns => [ @columns ], - rows => $rows, + rows => $#$filtered + 1, + numeric => { map { $_, $self->_is_numeric($_) } @columns }, + filters => $self->_current_filters, ); } +sub order { + my $self = shift; + $self->session('order', $self->param('order')); + $self->session('sort', $self->param('sort')); + $self->redirect_to('/data/items'); +} + +sub _is_numeric { + my ( $self, $name ) = @_; + + my $stats = $self->_loaded( 'stats' ); + + # sort facet numerically if more >50% elements are numeric + my $count = $stats->{$name}->{count}; + $count -= $stats->{$name}->{empty} if defined $stats->{$name}->{empty}; + defined $stats->{$name}->{numeric} && + $stats->{$name}->{numeric} > $count / 2; +} + +sub _remove_filter { + my ($self,$name) = @_; + warn "_remove_filter $name\n"; + + my $path = $self->session('path'); + + delete $filters->{$name}; + delete $loaded->{$path}->{filters}->{$name}; + warn "filters left: ", keys %{ $loaded->{$path}->{filters} }; + + foreach ( + grep { /\b$name\b/ } + keys %{ $loaded->{$path}->{filtered} } + ) { + delete $loaded->{$path}->{filtered}->{$_}; + warn "remove filtered cache $_"; + } +} + sub facet { my $self = shift; - if ( my $remove = $self->param('remove') ) { - my $f = $self->session('filters'); - delete $f->{$remove}; - $self->session( 'filters' => $f ); - $self->redirect_to( '/data/table' ); + my $path = $self->session('path') || $self->redirect_to( '/data/index' ); + + if ( my $name = $self->param('remove') ) { + $self->_remove_filter( $name ); + $self->redirect_to( '/data/items' ); } my $facet; my $name = $self->param('name') || die "no name"; - foreach my $i ( $self->_data_items ) { - next unless exists $i->{$name}; - if ( ref $i->{$name} eq 'ARRAY' ) { - $facet->{$_}++ foreach @{ $i->{$name} }; + my $all = $self->_perm_scalar('all', 1); + my $data = $self->_loaded('data'); + + my $filters = $self->_current_filters; + my $all_filters = join(' ',sort keys %$filters,'order:',$self->session('order')); + my $filtered = $loaded->{$path}->{filtered}->{$all_filters} + if defined $loaded->{$path}->{filtered}->{$all_filters}; + + if ( ! $filtered || $all ) { + $filtered = [ 0 .. $#{ $data->{items} } ]; + warn "filter all values\n"; + } else { + warn "filter using $all_filters\n"; + } + + foreach my $i ( @$filtered ) { + my $item = $data->{items}->[$i]; + if ( ! exists $item->{$name} ) { + $facet->{ _missing }++; + } elsif ( ref $item->{$name} eq 'ARRAY' ) { + $facet->{$_}++ foreach @{ $item->{$name} }; } else { - $facet->{ $i->{$name} }++; + $facet->{ $item->{$name} }++; } } # warn "# facet $name ",dump $facet; my $checked; - if ( my $f = $self->session('filters') ) { - if ( defined $f->{$name} ) { - $checked = $self->_checked( @{ $f->{$name} } ); - } - } - - my $sort = $self->param('sort') || 'c'; - - # sort facet numerically if more >50% elements are numeric - my $numeric = defined $stats->{$name}->{numeric} && - $stats->{$name}->{numeric} > $stats->{$name}->{count} / 2; - - my @facet_names = sort { - if ( $sort =~ m/a/i ) { - $numeric ? $a <=> $b : lc $a cmp lc $b; - } elsif ( $sort =~ m/d/i ) { - $numeric ? $b <=> $a : lc $b cmp lc $a; - } elsif ( $sort =~ m/c/i ) { - $facet->{$b} <=> $facet->{$a}; + my @facet_names = + $all ? keys %$facet + : defined $filters->{$name} ? @{ $filters->{$name} } + : keys %$facet; + + $checked = $self->_checked( @{ $filters->{$name} } ) if defined $filters->{$name}; + + my $numeric = $self->_is_numeric($name); + + my $sort = $self->param('sort'); + # sort numeric facets with more than 5 values ascending + $sort ||= $numeric && $#facet_names > 4 ? 'a' : 'c'; + + @facet_names = sort { + my $result; + if ( $sort eq 'a' ) { + $result = $numeric ? $a <=> $b : lc $a cmp lc $b; + } elsif ( $sort eq 'd' ) { + $result = $numeric ? $b <=> $a : lc $b cmp lc $a; + } elsif ( $sort eq 'c' ) { + $result = ( $facet->{$b} || -1 ) <=> ( $facet->{$a} || -1 ) } else { warn "unknown sort: $sort"; - $a cmp $b; + $result = $a cmp $b; } - } keys %$facet; + $result = $a cmp $b unless defined $result; # FIXME cludge for numeric facets with invalid data + $result; + } @facet_names; $self->render( name => $name, facet => $facet, checked => $checked, facet_names => \@facet_names, sort => $sort, numeric => $numeric, ); } -sub _checked { +sub edit { my $self = shift; - my $checked; - $checked->{$_}++ foreach @_; - warn "# _checked ",dump($checked); - return $checked; + my $content = $self->param('content'); + + $self->render( + content => $content + ); } 1;