From: Dobrica Pavlinusic <dpavlin@rot13.org>
Date: Tue, 19 Jul 2011 20:43:15 +0000 (+0000)
Subject: make dedup pass test and refactore JSON store
X-Git-Url: http://git.rot13.org/?a=commitdiff_plain;h=851afb3dbb37aa7b3a8c20244b2a9ccc038c4d94;p=cloudstore.git

make dedup pass test and refactore JSON store

- store directory data in dot files
- modify names of VFS calls
- show unused md5sum files queued for removal
---

diff --git a/lib/CloudStore/JSON.pm b/lib/CloudStore/JSON.pm
index c961350..b757be1 100644
--- a/lib/CloudStore/JSON.pm
+++ b/lib/CloudStore/JSON.pm
@@ -7,13 +7,16 @@ use JSON::XS;
 use File::Path qw();
 use File::Slurp qw();
 
-sub make_path {
-	warn "### VFS make_path $_[0]\n";
-	File::Path::make_path(@_);
+sub make_dir {
+	my ( $dir, $json ) = @_;
+	warn "### VFS make_dir $dir\n";
+	File::Path::make_path($dir);
+	$dir =~ s{/([^/]+)/?$}{/.$1};
+	File::Slurp::write_file $dir, $json if $json;
 }
 
-sub remove_path {
-	warn "### VFS remove_path $_[0]\n";
+sub remove_dir {
+	warn "### VFS remove_dir $_[0]\n";
 	File::Path::remove_tree(@_);
 }
 
@@ -23,13 +26,15 @@ sub remove_file {
 }
 
 sub write_file {
-	warn "### VFS write_file $_[0]\n";
+	warn "### VFS ", -e $_[0] ? 'modify' : 'write', "_file $_[0]\n";
 	File::Slurp::write_file(@_);
 }
 
-sub modify_existing {
-	warn "### VFS modify_existing $_[0]\n";
-	File::Slurp::write_file(@_);
+sub read_file {
+	my $path = shift;
+	warn "### VFS read_file $path\n";
+	$path =~ s{/([^/]+)/?$}{/.$1} if -d $path;
+	File::Slurp::read_file($path);
 }
 
 sub rsync_transfer {
@@ -40,27 +45,33 @@ sub rsync_transfer {
 
 	if ( $data->{itemize} =~ m/^[c>]([fd])/ ) { # received change/create
 		my $type = $1;
-		if ( -e $path ) {
-			modify_existing $path;
-		}
 
 		if ( $type eq 'f' ) {
 			if ( $path =~ m{^(.+)/[^/]+$} ) { # have dir
-				make_path $1 if ! -e $1;
+				make_dir $1, $json if ! -e $1;
 			}
 			write_file $path, $json;
 		} elsif ( $type eq 'd' ) {
-			make_path $path;
+			make_dir $path, $json;
+		} else {
+			die "unknown type $type";
 		}
 	} elsif ( $data->{itemize} =~ m/\*deleting/ ) {
 		if ( -d $path ) {
-			remove_path $path;
+			remove_dir $path;
 		} elsif ( -f $path ) {
 			remove_file $path;
+		} else {
+			warn "ignored delete $path";
 		}
 	}
 	return $json;
 }
 
+sub file_data {
+	my $path = shift;
+	$path =~ s{/blob/}{/json/};
+	return decode_json read_file $path;
+}
 
 1;
diff --git a/lib/CloudStore/dedup.pm b/lib/CloudStore/dedup.pm
index 59fe133..fafd5e8 100644
--- a/lib/CloudStore/dedup.pm
+++ b/lib/CloudStore/dedup.pm
@@ -3,6 +3,9 @@ use warnings;
 use strict;
 use autodie;
 
+use CloudStore::JSON;
+use Data::Dump qw(dump);
+
 sub path_md5 {
 	my ( $path, $md5 ) = @_;
 
@@ -23,9 +26,12 @@ sub path_md5 {
 sub path_remove {
 	my ( $path ) = @_;
 
-	my ( undef, undef, undef, $nlink ) = stat $path;
+	my $deleted = CloudStore::JSON::file_data( $path );
+
+	my ( undef, undef, undef, $nlink ) = stat "md5/$deleted->{md5}";
+
+	warn "path_remove $path $nlink ",dump $deleted;
 
-	warn "nlink $path $nlink";
 }
 
 my $empty_md5 = " " x 32;
diff --git a/rsync-piper.pl b/rsync-piper.pl
index ea87cef..2c93367 100755
--- a/rsync-piper.pl
+++ b/rsync-piper.pl
@@ -167,7 +167,7 @@ while(1) {
 
 			print ">>> data ",dump( \%data ) if $ENV{DEBUG};
 
-			CloudStore::dedup::data \%data;
+			CloudStore::dedup::data \%data; # uses deleted json files!
 
 			my $json = CloudStore::JSON::rsync_transfer \%data;
 
diff --git a/t/JSON.t b/t/JSON.t
index 4ff3f72..48e7a77 100755
--- a/t/JSON.t
+++ b/t/JSON.t
@@ -2,14 +2,14 @@
 use strict;
 use warnings;
 
-use Test::More tests => 3;
+use Test::More tests => 7;
 use Data::Dump qw(dump);
 
 use lib 'lib';
 
 use_ok 'CloudStore::JSON';
 
-ok( CloudStore::JSON::remove_path("users/test"), 'remove test users data' );
+ok( CloudStore::JSON::remove_dir("users/test"), 'remove test users data' );
 
 ok my $r = CloudStore::JSON::rsync_transfer(
 {
@@ -29,3 +29,9 @@ ok my $r = CloudStore::JSON::rsync_transfer(
 ), 'rsync_transfer';
 
 diag dump($r);
+
+ok $r = CloudStore::JSON::file_data('users/test/blob/dir-test/bar'), 'file_data';
+isa_ok $r, 'HASH';
+
+ok $r = CloudStore::JSON::file_data('users/test/blob/dir-test'), 'dir data';
+isa_ok $r, 'HASH';
diff --git a/test.sh b/test.sh
index f36ee9c..a6644d6 100755
--- a/test.sh
+++ b/test.sh
@@ -14,6 +14,7 @@ pull() {
 	RSYNC_PASSWORD=secret rsync rsync://test@127.0.0.1:6501/test/$from $to
 }
 
+./create-user.sh test secret
 
 dir=/tmp/test
 rm -Rf "$dir"
@@ -47,6 +48,9 @@ pull non-existing || true
 
 tail -20 $user_dir/log/*
 
-diff -urw $user_dir/blob $dir/
-diff -urw $user_dir/json $dir/
+#diff -urw $user_dir/blob $dir/
+#diff -urw $user_dir/json $dir/
 
+ls -al $user_dir/blob $user_dir/json
+
+#./remove-user.sh test