X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=lib%2FMojoFacets%2FData.pm;h=7344f891dcc40495fc304eff244b47202e54a711;hb=6ab85ad1235b0a2c187d7a50c621dcdaa5469fad;hp=17cd6701696331a503c2c08da601372794726bd9;hpb=4408a940ec7c8603de3ac892c8abb493de8f6f8d;p=MojoFacets.git

diff --git a/lib/MojoFacets/Data.pm b/lib/MojoFacets/Data.pm
index 17cd670..7344f89 100644
--- a/lib/MojoFacets/Data.pm
+++ b/lib/MojoFacets/Data.pm
@@ -13,9 +13,11 @@ use File::Find;
 use Storable;
 use Time::HiRes qw(time);
 use File::Path qw(mkpath);
+use Text::Unaccent::PurePerl;
 
 use MojoFacets::Import::File;
 use MojoFacets::Import::HTMLTable;
+use MojoFacets::Import::CSV;
 
 our $loaded;
 our $filters;
@@ -38,6 +40,9 @@ sub index {
 		} elsif ( -d $file && $file =~ m/\.html$/ ) {
 			$file =~ s/$data_dir\/*//;
 			push @files, $file;
+		} elsif ( -f $file && $file =~ m/\.csv$/i ) {
+			$file =~ s/$data_dir\/*//;
+			push @files, $file;
 		} else {
 			#warn "IGNORE: $file\n";
 		}
@@ -178,7 +183,11 @@ sub _load_path {
 
 	my $data;
 	if ( -f $full_path ) {
-		$data = MojoFacets::Import::File->new( full_path => $full_path, path => $path )->data;
+		if ( $full_path =~ m/.csv/i ) {
+			$data = MojoFacets::Import::CSV->new( full_path => $full_path )->data;
+		} else {
+			$data = MojoFacets::Import::File->new( full_path => $full_path, path => $path )->data;
+		}
 	} elsif ( -d $full_path && $full_path =~ m/.html/ ) {
 		$data = MojoFacets::Import::HTMLTable->new( dir => $full_path )->data;
 	} else {
@@ -244,7 +253,7 @@ sub load {
 	if ( ! defined $loaded->{$path}->{columns} ) {
 		my $columns_path = $self->_permanent_path( 'columns' );
 		if ( -e $columns_path ) {
-			my @columns = map { s/[\r\n]+$//; $_ } read_file $columns_path;
+			my @columns = map { s/[\r\n]+$//; $_ } read_file $columns_path, binmode => ':utf8';
 			$loaded->{$path}->{columns} = [ @columns ];
 			warn "# columns_path $columns_path ",dump(@columns);
 		} else {
@@ -317,7 +326,7 @@ sub _export_path {
 	}
 	my $dir = $self->app->home->rel_dir('public') . "/export/$path";
 	mkpath $dir unless -e $dir;
-	$dir . '/' . join('.', @_);
+	$dir . '/' . unac_string( join('.', @_) );
 }
 
 sub columns {
@@ -325,7 +334,7 @@ sub columns {
 
 	if ( $self->param('columns') ) {
 		my @columns = $self->_param_array('columns');
-		write_file( $self->_permanent_path( 'columns' ), map { "$_\n" } @columns );
+		write_file( $self->_permanent_path( 'columns' ), { binmode => ':utf8' }, map { "$_\n" } @columns );
 		$self->redirect_to('/data/items');
 	}
 
@@ -398,7 +407,7 @@ sub filter {
 		$self->_filter_on_data( $name, @vals );
 		if ( my $permanent = $self->param('_permanent') ) {
 			my $permanent_path = $self->_export_path( 'filter', $name, $permanent );
-			write_file $permanent_path, map { "$_\n" } @vals;
+			write_file $permanent_path, { binmode => ':utf8' }, map { "$_\n" } @vals;
 			warn "permanent filter $permanent_path ", -s $permanent_path;
 		}
 	}
@@ -604,7 +613,7 @@ sub items {
 	my $commit = $self->param('commit');
 	my $test = $self->param('test');
 
-	my $cols_changed;
+	my $commit_changed;
 
 	if ( $code && ( $test || $commit ) ) {
 		# XXX find columns used in code snippet and show them to user
@@ -614,22 +623,24 @@ sub items {
 				$column =~ s/$1$//;
 			}
 			next if $column =~ m/\$/; # hide columns with vars in them
-			$cols_changed->{$column} = 0;
+			$commit_changed->{$column} = 0;
 		}
 	}
 
 	my $code_path = $self->app->home->rel_dir('public') . "/code";
 	if ( $commit ) {
 
-		my $o = { map { $_ => 1 } grep { defined $loaded->{$path}->{stats}->{$_}->{count} } keys %{ $self->_loaded('stats') } };
-		#warn "XXX o ",dump( $o );
-
 		warn "# commit on ", $#$filtered + 1, " items:\n$code\n";
 		my $out;
 		foreach ( 0 .. $#$filtered ) {
 			my $i = $filtered->[$_];
 			my $row = $data->{items}->[$i];
+			my $update;
 			eval $code;
+			foreach ( keys %$update ) {
+				$commit_changed->{$_}++;
+				$row->{$_} = $update->{$_};
+			}
 		}
 		if ( my $description = $self->param('code_description') ) {
 			my $depends = $self->param('code_depends') || die "no code_depends?";
@@ -646,6 +657,7 @@ sub items {
 			my $commit_dataset = join('.'
 				, $self->param('code_depends')
 				, $self->param('code_description')
+				, time()
 			);
 			my $key = $self->param('code_depends');
 			$key =~ s/,.+$//;
@@ -653,7 +665,16 @@ sub items {
 			my $items;
 			foreach my $n ( keys %$out ) {
 				my $i = { $key => [ $n ] };
-				$i->{$_} = [ $out->{$n}->{$_} ] foreach keys %{ $out->{$n} };
+				my $ref = ref $out->{$n};
+				if ( $ref eq 'HASH' ) {
+					$i->{$_} = [ $out->{$n}->{$_} ] foreach keys %{ $out->{$n} };
+				} elsif ( $ref eq 'ARRAY' ) {
+					$i->{$_} = $out->{$n};
+				} elsif ( ! $ref ) {
+					$i->{value} = [ $out->{$n} ];
+				} else {
+					$i->{_error} = [ dump($out->{$n}) ];
+				}
 				push @$items, $i;
 			};
 			undef $out;
@@ -678,23 +699,22 @@ sub items {
 		}
 
 		# this might move before $out to recalculate stats on source dataset?
-		my $c = { map { $_ => 1 } @columns };
-		#warn "XXX c ",dump( $c );
-
 		__path_modified( $path, 2 );
-		$o->{$_}-- foreach keys %{ $self->_loaded('stats') };
-		#warn "XXX o ",dump( $o );
-		my @added_columns = grep { $o->{$_} && ! $c->{$_} } keys %$o;
+		my $c = { map { $_ => 1 } @columns };
+		my @added_columns = sort grep { ! $c->{$_} } keys %$commit_changed;
 		warn "# added_columns ",dump( @added_columns );
 		unshift @columns, @added_columns;
 
 		$self->session('columns', [ @columns ]);
 		$loaded->{$path}->{columns} = [ @columns ];
 		warn "# new columns ",dump( @columns );
+
+		__invalidate_path_column( $path, $_ ) foreach keys %$commit_changed;
 	}
 
 	my $sorted_items;
 	my $from_end = $sort eq 'd' ? $#$filtered : 0;
+	my $test_changed;
 	my $out;
 	foreach ( 0 .. $limit ) {
 		my $i = $_ + $offset;
@@ -702,36 +722,56 @@ sub items {
 		$i = $from_end - $i if $from_end;
 		my $id = $filtered->[$i];
 		my $row = Storable::dclone $data->{items}->[ $id ];
-		my $old = { map { $_ => 1 } keys %$row };
 		if ( $code && $test ) {
+			my $update;
 			eval $code;
 			if ( $@ ) {
 				warn "ERROR evaling\n$code\n$@";
 				$self->stash('eval_error', $@) if $@;
 			} else {
-				warn "EVAL ",dump($row);
-				$old->{$_}-- foreach keys %$row;
-				warn "columns changed ",dump($old);
-				$cols_changed->{$_}++ foreach grep { $old->{$_} == -1 } keys %$old;
+				warn "EVAL ",dump($update);
+				foreach ( keys %$update ) {
+					$test_changed->{$_}++;
+					$row->{$_} = $update->{$_};
+				}
 			}
 		}
 		$row->{_row_id} ||= $id;
 		push @$sorted_items, $row;
 	}
 
-	my @added_columns = sort grep { $cols_changed->{$_} > 0 } keys %$cols_changed;
+	if ( $self->param('export') ) {
+		my $export_path = $self->_export_path( 'items', @columns);
+		open(my $fh, '>', $export_path) || warn "ERROR: can't open $export_path: $!";
+		foreach my $f ( 0 .. $#$filtered ) {
+			print $fh join("\t", map {
+				my $i = $data->{items}->[ $filtered->[$f] ];
+				if ( ref $i->{$_} eq 'ARRAY' ) {
+					join(',', @{ $i->{$_} });
+				} else {
+					dump $i->{$_};
+				}
+			} @columns),"\n";
+		}
+		close($fh);
+		warn "export $export_path ", -s $export_path, " bytes\n";
+	}
+
+	warn "# test_changed ",dump( $test_changed );
+	my $c = { map { $_ => 1 } @columns };
+	my @added_columns = sort grep { ! $c->{$_} } keys %$test_changed;
 	unshift @columns, @added_columns;
 
 	warn "# sorted_items ", $#$sorted_items + 1, " offset $offset limit $limit order $sort";
 
 	my $code_depends = $self->param('code_depends')||
-	join(',', sort grep { $cols_changed->{$_} == 0 } keys %$cols_changed );
+	join(',', sort grep { $test_changed->{$_} == 0 } keys %$test_changed );
 	my $code_description = $self->param('code_description') ||
 	join(',', @added_columns);
 
 	$code_depends ||= $code_description; # self-modifing
 
-	warn "# cols_changed ",dump( $cols_changed, $code_depends, $code_description );
+	warn "# test_changed ",dump( $test_changed, $code_depends, $code_description );
 
 	$self->render(
 		order => $order,
@@ -744,7 +784,7 @@ sub items {
 		unique  => { map { $_, $self->_is_unique( $_) } @columns },
 		filters => $self->_current_filters,
 		code => $code,
-		cols_changed => $cols_changed,
+		cols_changed => $commit ? $commit_changed : $test_changed,
 		code_depends => $code_depends,
 		code_description => $code_description,
 		code_path => $code_path,
@@ -993,7 +1033,7 @@ sub export {
 		if ( $import =~ m{/filter\.(.+?)\..+} ) {
 			my $name = $1;
 			my @vals = map { chomp; $_ }
-				read_file $self->app->home->rel_dir('public') . "/export/$import";
+				read_file $self->app->home->rel_dir('public') . "/export/$import", binmode => ':utf8';
 			$self->_remove_filter( $name );
 			$self->_filter_on_data( $name, @vals );
 			$self->session( 'offset' => 0 );
@@ -1003,9 +1043,10 @@ sub export {
 		}
 	}
 
-	$self->render( export => [
-		glob( $self->_export_path . '*' )
-	] );
+	my @files = grep { ! /\.png$/ } glob( $self->_export_path . '*' );
+	my $mtime = { map { $_ => (stat($_))[9] } @files };
+	@files = sort { $mtime->{$b} <=> $mtime->{$a} } @files;
+	$self->render( export => [ @files ] );
 }
 
 sub __loaded_paths {