make dedup pass test and refactore JSON store

- store directory data in dot files
- modify names of VFS calls
- show unused md5sum files queued for removal

diff --git a/lib/CloudStore/JSON.pm b/lib/CloudStore/JSON.pm
index c961350..b757be1 100644 (file)
--- a/lib/CloudStore/JSON.pm
+++ b/lib/CloudStore/JSON.pm
@@ -7,13 +7,16 @@ use JSON::XS;
 use File::Path qw();
 use File::Slurp qw();
 
-sub make_path {
-       warn "### VFS make_path $_[0]\n";
-       File::Path::make_path(@_);
+sub make_dir {
+       my ( $dir, $json ) = @_;
+       warn "### VFS make_dir $dir\n";
+       File::Path::make_path($dir);
+       $dir =~ s{/([^/]+)/?$}{/.$1};
+       File::Slurp::write_file $dir, $json if $json;
 }
 
-sub remove_path {
-       warn "### VFS remove_path $_[0]\n";
+sub remove_dir {
+       warn "### VFS remove_dir $_[0]\n";
        File::Path::remove_tree(@_);
 }
 
@@ -23,13 +26,15 @@ sub remove_file {
 }
 
 sub write_file {
-       warn "### VFS write_file $_[0]\n";
+       warn "### VFS ", -e $_[0] ? 'modify' : 'write', "_file $_[0]\n";
        File::Slurp::write_file(@_);
 }
 
-sub modify_existing {
-       warn "### VFS modify_existing $_[0]\n";
-       File::Slurp::write_file(@_);
+sub read_file {
+       my $path = shift;
+       warn "### VFS read_file $path\n";
+       $path =~ s{/([^/]+)/?$}{/.$1} if -d $path;
+       File::Slurp::read_file($path);
 }
 
 sub rsync_transfer {
@@ -40,27 +45,33 @@ sub rsync_transfer {
 
        if ( $data->{itemize} =~ m/^[c>]([fd])/ ) { # received change/create
                my $type = $1;
-               if ( -e $path ) {
-                       modify_existing $path;
-               }
 
                if ( $type eq 'f' ) {
                        if ( $path =~ m{^(.+)/[^/]+$} ) { # have dir
-                               make_path $1 if ! -e $1;
+                               make_dir $1, $json if ! -e $1;
                        }
                        write_file $path, $json;
                } elsif ( $type eq 'd' ) {
-                       make_path $path;
+                       make_dir $path, $json;
+               } else {
+                       die "unknown type $type";
                }
        } elsif ( $data->{itemize} =~ m/\*deleting/ ) {
                if ( -d $path ) {
-                       remove_path $path;
+                       remove_dir $path;
                } elsif ( -f $path ) {
                        remove_file $path;
+               } else {
+                       warn "ignored delete $path";
                }
        }
        return $json;
 }
 
+sub file_data {
+       my $path = shift;
+       $path =~ s{/blob/}{/json/};
+       return decode_json read_file $path;
+}
 
 1;

diff --git a/lib/CloudStore/dedup.pm b/lib/CloudStore/dedup.pm
index 59fe133..fafd5e8 100644 (file)
--- a/lib/CloudStore/dedup.pm
+++ b/lib/CloudStore/dedup.pm
@@ -3,6 +3,9 @@ use warnings;
 use strict;
 use autodie;
 
+use CloudStore::JSON;
+use Data::Dump qw(dump);
+
 sub path_md5 {
        my ( $path, $md5 ) = @_;
 
@@ -23,9 +26,12 @@ sub path_md5 {
 sub path_remove {
        my ( $path ) = @_;
 
-       my ( undef, undef, undef, $nlink ) = stat $path;
+       my $deleted = CloudStore::JSON::file_data( $path );
+
+       my ( undef, undef, undef, $nlink ) = stat "md5/$deleted->{md5}";
+
+       warn "path_remove $path $nlink ",dump $deleted;
 
-       warn "nlink $path $nlink";
 }
 
 my $empty_md5 = " " x 32;

diff --git a/rsync-piper.pl b/rsync-piper.pl
index ea87cef..2c93367 100755 (executable)
--- a/rsync-piper.pl
+++ b/rsync-piper.pl
@@ -167,7 +167,7 @@ while(1) {
 
                        print ">>> data ",dump( \%data ) if $ENV{DEBUG};
 
-                       CloudStore::dedup::data \%data;
+                       CloudStore::dedup::data \%data; # uses deleted json files!
 
                        my $json = CloudStore::JSON::rsync_transfer \%data;
 

diff --git a/t/JSON.t b/t/JSON.t
index 4ff3f72..48e7a77 100755 (executable)
--- a/t/JSON.t
+++ b/t/JSON.t
@@ -2,14 +2,14 @@
 use strict;
 use warnings;
 
-use Test::More tests => 3;
+use Test::More tests => 7;
 use Data::Dump qw(dump);
 
 use lib 'lib';
 
 use_ok 'CloudStore::JSON';
 
-ok( CloudStore::JSON::remove_path("users/test"), 'remove test users data' );
+ok( CloudStore::JSON::remove_dir("users/test"), 'remove test users data' );
 
 ok my $r = CloudStore::JSON::rsync_transfer(
 {
@@ -29,3 +29,9 @@ ok my $r = CloudStore::JSON::rsync_transfer(
 ), 'rsync_transfer';
 
 diag dump($r);
+
+ok $r = CloudStore::JSON::file_data('users/test/blob/dir-test/bar'), 'file_data';
+isa_ok $r, 'HASH';
+
+ok $r = CloudStore::JSON::file_data('users/test/blob/dir-test'), 'dir data';
+isa_ok $r, 'HASH';

diff --git a/test.sh b/test.sh
index f36ee9c..a6644d6 100755 (executable)
--- a/test.sh
+++ b/test.sh
@@ -14,6 +14,7 @@ pull() {
        RSYNC_PASSWORD=secret rsync rsync://test@127.0.0.1:6501/test/$from $to
 }
 
+./create-user.sh test secret
 
 dir=/tmp/test
 rm -Rf "$dir"
@@ -47,6 +48,9 @@ pull non-existing || true
 
 tail -20 $user_dir/log/*
 
-diff -urw $user_dir/blob $dir/
-diff -urw $user_dir/json $dir/
+#diff -urw $user_dir/blob $dir/
+#diff -urw $user_dir/json $dir/
 
+ls -al $user_dir/blob $user_dir/json
+
+#./remove-user.sh test