fist dedup implementation

diff --git a/lib/CloudStore/dedup.pm b/lib/CloudStore/dedup.pm
new file mode 100644 (file)
index 0000000..59fe133
--- /dev/null
+++ b/lib/CloudStore/dedup.pm
@@ -0,0 +1,47 @@
+package CloudStore::dedup;
+use warnings;
+use strict;
+use autodie;
+
+sub path_md5 {
+       my ( $path, $md5 ) = @_;
+
+       my $pool = 'md5'; # FIXME sharding?
+       mkdir $pool unless -e $pool;
+
+       if ( -e "$pool/$md5" ) {
+               warn "dedup hit $md5 $path\n";
+               my $dedup = $path . '.dedup';
+               rename $path, $dedup;
+               link "$pool/$md5", $path;
+               unlink $dedup;
+       } else {
+               link $path, "$pool/$md5";
+       }
+}
+
+sub path_remove {
+       my ( $path ) = @_;
+
+       my ( undef, undef, undef, $nlink ) = stat $path;
+
+       warn "nlink $path $nlink";
+}
+
+my $empty_md5 = " " x 32;
+
+sub data {
+       my $data = shift;
+
+       my $path = "users/$data->{login}/blob/$data->{file}";
+
+       return if -d $path;
+
+       if ( $data->{md5} ne $empty_md5 ) {
+               path_md5 $path => $data->{md5};
+       } else {
+               path_remove $path;
+       }
+}
+
+1;

diff --git a/rsync-piper.pl b/rsync-piper.pl
index 12e1e79..ea87cef 100755 (executable)
--- a/rsync-piper.pl
+++ b/rsync-piper.pl
@@ -12,6 +12,7 @@ use English;
 
 use lib 'lib';
 use CloudStore::JSON;
+use CloudStore::dedup;
 
 my $dir   = $ENV{RSYNC_DIR}  || '/srv/cloudstore';
 my $port  = $ENV{RSYNC_PORT} || 6501;
@@ -166,6 +167,8 @@ while(1) {
 
                        print ">>> data ",dump( \%data ) if $ENV{DEBUG};
 
+                       CloudStore::dedup::data \%data;
+
                        my $json = CloudStore::JSON::rsync_transfer \%data;
 
                        $gearman->dispatch_background( 'rsync_transfer' => $json );

diff --git a/t/dedup.t b/t/dedup.t
new file mode 100755 (executable)
index 0000000..54357e8
--- /dev/null
+++ b/t/dedup.t
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+
+use Test::More tests => 3;
+use Data::Dump qw(dump);
+
+use lib 'lib';
+
+use_ok 'CloudStore::dedup';
+
+my $md5 = "51ce99ec40129bfe1fd11d65b346d15e";
+my $file = "users/test/blob/foo";
+
+my $r;
+
+ok( $r = CloudStore::dedup::path_md5( $file, "test1-$md5" ), 'path_md5' );
+diag dump($r);
+
+ok( $r = CloudStore::dedup::path_remove( $file ), 'path_remove' );
+diag dump($r);
+

diff --git a/test.sh b/test.sh
index 9b069d7..f36ee9c 100755 (executable)
--- a/test.sh
+++ b/test.sh
@@ -31,6 +31,9 @@ pull bar baz
 push baz -v
 ps ax >> baz
 push baz -v
+cp baz baz2
+push baz baz2 -v
+
 mkdir dir-test
 # deduplicate?
 cp foo bar dir-test/