X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=lib%2FMojoFacets%2FData.pm;h=761ded7e8513fbf9025b41ca590207e74684881d;hb=69082e371e2a8fc717f79ce308c7c045ddd72101;hp=a8dc7739b7dad10ca98e3cecddefa5341f199049;hpb=51d00883ba7487327695800beb1aeaca97c942f9;p=MojoFacets.git

diff --git a/lib/MojoFacets/Data.pm b/lib/MojoFacets/Data.pm
index a8dc773..761ded7 100644
--- a/lib/MojoFacets/Data.pm
+++ b/lib/MojoFacets/Data.pm
@@ -13,6 +13,7 @@ use File::Find;
 use Storable;
 use Time::HiRes qw(time);
 use File::Path qw(mkpath);
+use Text::Unaccent::PurePerl;
 
 use MojoFacets::Import::File;
 use MojoFacets::Import::HTMLTable;
@@ -278,11 +279,18 @@ sub _loaded {
 			warn "rebuild stats for $path ignored caller $caller\n";
 		} else {
 			warn "rebuild stats for $path FORCED by modified caller $caller\n";
-			$loaded->{$path}->{stats} = __stats( $loaded->{$path}->{data}->{items} );
+#			$loaded->{$path}->{stats} = __stats( $loaded->{$path}->{data}->{items} );
+			$loaded->{$path}->{rebuild_stats} = 1;
 			$loaded->{$path}->{modified} = 1;
 		}
 	}
 
+	if ( defined $loaded->{$path}->{rebuild_stats} ) {
+		warn "rebuild_stats $path";
+		$loaded->{$path}->{stats} = __stats( $loaded->{$path}->{data}->{items} );
+		delete $loaded->{$path}->{rebuild_stats};
+	}
+
 	if ( ! defined $loaded->{$path}->{$name} ) {
 		warn "$path $name isn't loaded\n";
 		$self->_load_path( $path );
@@ -316,6 +324,22 @@ sub _permanent_path {
 	$self->app->home->rel_dir('data') . '/' . join('.', $path, @_);
 }
 
+sub __unac {
+	my $n = shift;
+	$n = unac_string($n);
+	$n =~ s/\W+/_/g;
+	return $n;
+}
+
+sub _column_from_unac {
+	my ($self,$name) = @_;
+
+	my $stats = $self->_loaded('stats');
+	my $cols_norm = { map { __unac( $_ ) => $_ } keys %$stats };
+
+	$cols_norm->{$name} || die "can't find column $name in ", dump($cols_norm);
+}
+
 sub _export_path {
 	my $self = shift;
 	my $path = $self->_param_or_session('path');
@@ -325,7 +349,10 @@ sub _export_path {
 	}
 	my $dir = $self->app->home->rel_dir('public') . "/export/$path";
 	mkpath $dir unless -e $dir;
-	$dir . '/' . join('.', @_);
+	my $name = join('.', map { __unac($_) } @_ );
+	my $full = $dir . '/' . $name;
+	$full =~ s/\/+$// if -d $full; # strip trailing slash for dirs
+	return $full;
 }
 
 sub columns {
@@ -464,15 +491,12 @@ sub _filter_on_data {
 sub _current_filters {
 	my $self = shift;
 	my $current_filters;
-	my $columns = $self->_loaded('header');
-	if ( my $sc = $self->session('columns') ) {
-		$columns = $sc;
-	}
+	my $stats = $self->_loaded('stats');
 
 	$current_filters->{ $_ } = $filters->{ $_ }
-		foreach ( grep { defined $filters->{ $_ } } @$columns )
+		foreach ( grep { defined $filters->{ $_ } } keys %$stats )
 	;
-	warn "# _current_filters ",dump($columns);
+	warn "# _current_filters ",dump( keys %$current_filters );
 	return $current_filters;
 }
 
@@ -525,6 +549,21 @@ sub __all_filters {
 	join(',', sort(@_), 'order', $order);
 }
 
+sub __commit_path_code {
+	my ( $path, $i, $code, $commit_changed ) = @_;
+
+	my $items = $loaded->{$path}->{data}->{items} || die "no items for $path";
+	my $row = $items->[$i];
+	my $update;
+	eval $code;
+	foreach ( keys %$update ) {
+		$$commit_changed->{$_}++;
+		$loaded->{$path}->{data}->{items}->[$i]->{$_} = $update->{$_};
+	}
+	warn "XX ",dump( $loaded->{$path}->{data}->{items}->[$i] );
+	warn "__commit_path_code $path ",dump( $update );
+}
+
 sub items {
 	my $self = shift;
 
@@ -607,7 +646,7 @@ sub items {
 	my $data = $self->_loaded('data');
 
 	my $code = $self->_param_scalar('code','');
-	$code =~ s{[\r\n]+$}{}s;
+	$code =~ s{[\r\n]+$}{\n}s;
 
 	my $commit = $self->param('commit');
 	my $test = $self->param('test');
@@ -633,24 +672,29 @@ sub items {
 		my $out;
 		foreach ( 0 .. $#$filtered ) {
 			my $i = $filtered->[$_];
-			my $row = $data->{items}->[$i];
-			my $update;
-			eval $code;
-			foreach ( keys %$update ) {
-				$commit_changed->{$_}++;
-				$row->{$_} = $update->{$_};
-			}
+			__commit_path_code( $path, $i, $code, \$commit_changed );
 		}
+
+		$self->_save_change({
+			path => $path,
+			time => $self->param('time') || time(),
+			user => $self->param('user') || $ENV{'LOGNAME'},
+			code => $code,
+			commit_changed => $commit_changed,
+		});
+
 		if ( my $description = $self->param('code_description') ) {
 			my $depends = $self->param('code_depends') || die "no code_depends?";
 			my $path = "$code_path/$depends.$description.pl";
 			if ( -e $path && ! $self->param('overwrite') ) {
 				warn "# code $path not saved\n";
 			} else {
-				write_file(  $path, { binmode => ':utf8' }, $code );
+				write_file(  $path, { binmode => ':utf8' }, "$code\n" );
 				warn "code $path ", -s $path, " bytes saved\n";
 			}
 		}
+
+		# remove console
 		$code = '';
 		if ( $out ) {
 			my $commit_dataset = join('.'
@@ -698,7 +742,7 @@ sub items {
 		}
 
 		# this might move before $out to recalculate stats on source dataset?
-		__path_modified( $path, 2 );
+		__path_rebuild_stats( $path );
 		my $c = { map { $_ => 1 } @columns };
 		my @added_columns = sort grep { ! $c->{$_} } keys %$commit_changed;
 		warn "# added_columns ",dump( @added_columns );
@@ -740,16 +784,21 @@ sub items {
 	}
 
 	if ( $self->param('export') ) {
-		my $export_path = "public/export/$path/" . join('.', @columns);
+		my $export_path = $self->_export_path( 'items', @columns);
 		open(my $fh, '>', $export_path) || warn "ERROR: can't open $export_path: $!";
 		foreach my $f ( 0 .. $#$filtered ) {
 			print $fh join("\t", map {
 				my $i = $data->{items}->[ $filtered->[$f] ];
-				if ( ref $i->{$_} eq 'ARRAY' ) {
-					join(',', @{ $i->{$_} });
+				my $v = '\N';
+				if ( ! defined $i->{$_} ) {
+					# nop
+				} elsif ( ref $i->{$_} eq 'ARRAY' ) {
+					$v =join(',', @{ $i->{$_} });
+					$v = '\N' if length($v) == 0;
 				} else {
-					dump $i->{$_};
+					$v = dump $i->{$_};
 				}
+				$v;
 			} @columns),"\n";
 		}
 		close($fh);
@@ -937,6 +986,19 @@ sub __path_modified {
 	warn "# __path_modified $path $value\n";
 }
 
+sub __path_rebuild_stats { $loaded->{ $_[0] }->{rebuild_stats} = 1 };
+
+sub _save_change {
+	my ($self,$change) = @_;
+
+	my $change_path = $self->_permanent_path( 'changes' );
+	mkdir $change_path unless -d $change_path;
+	$change_path .= '/' . $change->{time};
+	store $change, $change_path;
+	utime $change->{time}, $change->{time}, $change_path;
+	warn "_save_change $change_path ", dump($change);
+}
+
 sub edit {
 	my $self = shift;
 	my $new_content = $self->param('new_content');
@@ -965,7 +1027,7 @@ sub edit {
 		if ( $old ne $new
 			&& ! ( $old eq 'undef' && length($new_content) == 0 ) # new value empty, previous undef
 		) {
-			my $change = {
+			$self->_save_change({
 				path => $path,
 				column => $name,
 				pos => $i,
@@ -978,13 +1040,7 @@ sub edit {
 					grep { defined $loaded->{$path}->{stats}->{$_}->{unique} }
 					keys %{ $loaded->{$path}->{stats} }
 				},
-			};
-			my $change_path = $self->_permanent_path( 'changes' );
-			mkdir $change_path unless -d $change_path;
-			$change_path .= '/' . $change->{time};
-			store $change, $change_path;
-			utime $change->{time}, $change->{time}, $change_path;
-			warn "# $change_path ", dump($change);
+			});
 
 			warn "# change $path $i $old -> $new\n";
 			$loaded->{$path}->{data}->{items}->[$i]->{$name} = $v;
@@ -992,8 +1048,7 @@ sub edit {
 			__invalidate_path_column( $path, $name );
 
 			$status = 201; # created
-			# modified = 2 -- force rebuild of stats
-			__path_modified( $path, 2 );
+			__path_rebuild_stats( $path );
 	
 			$new_content = join("\xB6",@$v);
 
@@ -1027,12 +1082,16 @@ sub save {
 sub export {
 	my $self = shift;
 
+	my $dir = $self->app->home->rel_dir('public');
+
 	if ( my $import = $self->param('import') ) {
 
 		if ( $import =~ m{/filter\.(.+?)\..+} ) {
-			my $name = $1;
+			my $name = $self->_column_from_unac( $1 );
+
 			my @vals = map { chomp; $_ }
-				read_file $self->app->home->rel_dir('public') . "/export/$import", binmode => ':utf8';
+				read_file "$dir/export/$import", binmode => ':utf8';
+
 			$self->_remove_filter( $name );
 			$self->_filter_on_data( $name, @vals );
 			$self->session( 'offset' => 0 );
@@ -1042,9 +1101,19 @@ sub export {
 		}
 	}
 
-	$self->render( export => [
-		glob( $self->_export_path . '*' )
-	] );
+	if ( my $remove = $self->param('remove') ) {
+		my $path = "$dir/export/$remove";
+		unlink $path if -e $path;
+		$path .= '.png';
+		unlink $path if -e $path;
+	}
+
+	my $path = $self->_export_path || $self->redirect_to('/data/index');
+
+	my @files = grep { ! /\.png$/ } glob "$path/*";
+	my $mtime = { map { $_ => (stat($_))[9] } @files };
+	@files = sort { $mtime->{$b} <=> $mtime->{$a} } @files;
+	$self->render( export => [ @files ] );
 }
 
 sub __loaded_paths {