generate BerkeleyDB md5 hash, remove dependencies

[cloudstore.git] / lib / CloudStore / dedup.pm

diff --git a/lib/CloudStore/dedup.pm b/lib/CloudStore/dedup.pm
index fafd5e8..04488cd 100644 (file)
--- a/lib/CloudStore/dedup.pm
+++ b/lib/CloudStore/dedup.pm
@@ -4,6 +4,7 @@ use strict;
 use autodie;
 
 use CloudStore::JSON;
+use File::Path qw(make_path);
 use Data::Dump qw(dump);
 
 sub path_md5 {
@@ -24,14 +25,18 @@ sub path_md5 {
 }
 
 sub path_remove {
-       my ( $path ) = @_;
+       my $path = shift;
 
        my $deleted = CloudStore::JSON::file_data( $path );
 
-       my ( undef, undef, undef, $nlink ) = stat "md5/$deleted->{md5}";
+       my $md5_path = "md5/$deleted->{md5}";
 
-       warn "path_remove $path $nlink ",dump $deleted;
+       my ( undef, undef, undef, $nlink ) = stat $md5_path;
 
+       warn "path_remove $path $nlink ",dump $deleted;
+       open(my $fh, '>>', 'var/dedup.log');
+       print $fh join(' ', $md5_path, $nlink, @_), $/;
+       close $fh;
 }
 
 my $empty_md5 = " " x 32;
@@ -39,14 +44,40 @@ my $empty_md5 = " " x 32;
 sub data {
        my $data = shift;
 
-       my $path = "users/$data->{login}/blob/$data->{file}";
+       my $blob = "users/$data->{login}/blob";
+       my $path = "$blob/$data->{file}";
 
        return if -d $path;
 
+       if ( $data->{file} =~ /^(.+\/)?md5sum$/ ) {
+               my $dir = $1;
+               my $imported = 0;
+               warn "$path import\n";
+               open(my $md5sum, '<', $path);
+               while(<$md5sum>) {
+                       chomp;
+                       my ( $md5, $file ) = split(/\s+/,$_,2);
+                       if ( ! -e "md5/$md5" ) {
+                               warn "MISSING $md5 $file\n";
+                               next;
+                       }
+                       my $new = "users/$data->{login}/blob/$dir$file";
+                       if ( ! -e $new ) {
+                               # create path from md5sum file
+                               my $dir = $1 if $new =~ m{^(.+)/[^/]+$};
+                               make_path $dir unless -d $dir;
+                               $imported += link "md5/$md5", $new;
+                       } else {
+                               path_md5 $new => $md5;
+                       }
+               }
+               print "INFO: $path imported $imported files\n";
+       }
+
        if ( $data->{md5} ne $empty_md5 ) {
                path_md5 $path => $data->{md5};
        } else {
-               path_remove $path;
+               path_remove $path, $data->{timestamp};
        }
 }