don't die on md5sum without filename

[cloudstore.git] / lib / CloudStore / Store.pm

diff --git a/lib/CloudStore/Store.pm b/lib/CloudStore/Store.pm
index 924a4ad..1956a73 100644 (file)
--- a/lib/CloudStore/Store.pm
+++ b/lib/CloudStore/Store.pm
@@ -3,7 +3,7 @@ use warnings;
 use strict;
 
 use lib 'lib';
-use base 'CloudStore::MD5sum';
+use base qw(CloudStore::MD5sum);
 use CloudStore::API;
 
 use autodie;
@@ -31,13 +31,7 @@ sub new {
        return $self;
 }
 
-sub user_set {
-       my ( $self,$data ) = @_;
-}
-
-sub user_get {
-       my ( $self,$data ) = @_;
-}
+sub api { $_[0]->{api} }
 
 sub mkbasedir {
        my $dir = shift;
@@ -48,6 +42,8 @@ sub mkbasedir {
 sub modify_file {
        my ( $self,$data ) = @_;
 
+=for removed
+
        if ( $data->{file} =~ m{^(.*/)?.sync/send/([^/]+)$} ) {
                my $from_dir = $1;
                warn "SEND $2 from $from_dir\n";
@@ -130,6 +126,7 @@ sub modify_file {
 
                        $self->md5sum($data)->out( $from_dir . $from );
                        $self->md5sum($data)->put( $from_dir . $to => $md5 );
+                       $self->md5sum_close($data);
 
                        warn "$md5 moved to $from_dir $to";
                }
@@ -137,6 +134,13 @@ sub modify_file {
                return 0; # skip dedup
        }
 
+=cut
+
+       if ( $data->{file} =~ m{^(.*/)?.sync/} ) {
+               # ignore .sync/ files from client
+               return 0;
+       }
+
        #return $file->{size} > 4096 ? 1 : 0; # FIXME
        return 1; # dedup
 }
@@ -147,10 +151,13 @@ sub new_file {
 #      $self->file_set($data);
 }
 
+# client doesn't issue --delete
 sub removed_file {
        my ( $self, $data ) = @_;
 
-       my $md5 = $self->md5sum($data)->get( $data->{file} );
+=for removed
+       my $path = $self->blob_path( $data );
+       my $md5 = $self->md5_get( $path );
        return unless $md5; # directories don't have md5sums
        my $path = $self->{md5pool} . '/' . $md5;
        my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
@@ -166,7 +173,9 @@ sub removed_file {
                warn "# chown $id $gid $path";
        }
 
-       $self->md5sum($data)->out( $data->{file} );
+       $self->api->append_meta('md5sum', $user, 'delete', $data->{file} );
+=cut
+
 }
 
 sub make_dir {
@@ -179,6 +188,8 @@ sub new_link {
 
        warn "# new_link ",dump $data;
 
+=for removed
+
        if ( $data->{file} =~ m{^(.*/?)\.send/([^/]+)/(.+)$} ) {
                my ( $dir, $to, $name ) = ( $1, $2, $3 );
                my $path = $self->blob_path($data);
@@ -227,6 +238,9 @@ sub new_link {
                        warn "ERROR: can't SEND To:$to Name:$name Link:$link_to";
                }
        }
+
+=cut
+
 }
 
 sub init_pid_login {
@@ -241,8 +255,6 @@ sub init_pid_login {
 sub cleanup_pid {
        my ( $self, $pid ) = @_;
 
-       $self->md5sum_close;
-
        delete $self->{pid}->{$pid};
        warn "removed $pid";
 }
@@ -256,13 +268,18 @@ sub rsync_log {
                my $pid = $1;
                $self->cleanup_pid( $pid );
        } else {
-               warn "## rsync_log $data";
+               warn "## rsync_log $data" if $ENV{DEBUG};
        }
 }
 
 sub blob_path {
        my ( $self, $data, $path ) = @_;
-       my $blob = $self->{pid}->{ $data->{pid} }->{dir} || die "no dir for $data->{pid} in ",dump( $self->{pid} );
+       my $blob = $self->{pid}->{ $data->{pid} }->{dir};
+       if ( ! $blob ) {
+               warn "ERROR: $data->{pid} not found, possible restart?";
+               $self->init_pid_login( $data->{pid}, $data->{login} );
+               $blob = $self->{pid}->{ $data->{pid} }->{dir} || die "no dir for ", dump( $self->{pid}->{ $data->{pid} } );
+       }
        $blob .= '/' . ( defined $path ? $path : $data->{file} );
        return $blob;
 }
@@ -286,14 +303,21 @@ sub rsync_transfer {
                } else {
                        die "unknown type $type ", dump $data;
                }
+               $self->api->refresh_file_list( $data->{login} );
        } elsif ( $data->{itemize} =~ m/\*deleting/ ) {
                $self->removed_file($data);
+               $self->api->refresh_file_list( $data->{login} );
        } else {
                warn "IGNORED ",dump($data) if $ENV{DEBUG};
        }
        return $data;
 }
 
+sub append {
+       my $self = shift @_;
+       $self->{api}->append( @_ );
+}
+
 sub md5pool {
        my ( $self, $data ) = @_;
 
@@ -303,6 +327,11 @@ sub md5pool {
        my $md5 = $data->{md5} || die "no md5 in ",dump $data;
        my $path = $self->blob_path($data);
 
+       if ( ! -e $path ) {
+               warn "ERROR missing path $path";
+               return;
+       }
+
        my $pool_md5 = "$pool/$md5";
 
        if ( -e $pool_md5 ) {
@@ -311,13 +340,15 @@ sub md5pool {
                my ($pool_uid,$pool_size) = (stat($pool_md5))[4,7];
                my $user = $self->{api}->user_info( $data->{login} );
 
-               if ( $pool_uid != $self->{api}->{md5}->{uid} ) {
-                       chown $self->{api}->{md5}->{uid}, $self->{api}->{md5}->{gid}, $pool_md5;
-                       chmod oct("0444"), $pool_md5;
-                       my $steal_user = $self->{api}->user_info( $pool_uid );
-                       $self->append( $steal_user, 'dedup-steal', $pool_size, $pool_uid, $data->{file} );
+               if ( $pool_uid != $user->{uid} ) {
+                       if ( $pool_uid != $self->{api}->{md5}->{uid} ) {
+                               chown $self->{api}->{md5}->{uid}, $self->{api}->{md5}->{gid}, $pool_md5;
+                               chmod oct("0444"), $pool_md5;
+                               my $steal_user = $self->{api}->user_info( $pool_uid );
+                               $self->append( $steal_user, 'dedup-steal', $pool_size, $pool_uid, $data->{file} );
+                       }
+                       $self->append( $user, 'dedup', $pool_size, $pool_uid, $data->{file} );
                }
-               $self->append( $user, 'dedup', $pool_size, $pool_uid, $data->{file} );
 
                my $dedup = $path . '.dedup';
                rename $path, $dedup;
@@ -328,7 +359,9 @@ sub md5pool {
                warn "dedup +++ $md5 $path";
        }
 
-       $self->md5sum($data)->put( $data->{file} => $md5 );
+       $self->md5_set( $path => $md5 );
+
+       $self->api->append_meta('md5sum', $data->{login}, $md5, $data->{file} );
 }
 
 my $empty_md5 = " " x 32;
@@ -337,18 +370,23 @@ sub dedup {
        my ( $self, $data, $path ) = @_;
 
        if ( $data->{file} =~ /^(.+\/)?md5sum$/ ) {
-               my $dir = $1;
+               my $dir = $1 || '';
                my $imported = 0;
                warn "IMPORT ", $data->{file}, "\n";
                open(my $md5sum, '<', $path);
                while(<$md5sum>) {
                        chomp;
                        my ( $md5, $file ) = split(/\s+/,$_,2);
-                       if ( ! -e "$self->{md5path}/$md5" ) {
+                       if ( ! $file ) {
+                               warn "IGNORE $md5 without file\n";
+                               next;
+                       }
+                       if ( ! -e "$self->{md5pool}/$md5" ) {
                                warn "MISSING $md5 $file\n";
                                next;
                        }
                        my $new = {
+                               login => $data->{login},
                                pid => $data->{pid},
                                file => "$dir$file",
                                md5 => $md5,
@@ -361,11 +399,12 @@ sub dedup {
                                $self->md5pool( $new );
                        }
                }
-               print "INFO imported $imported files from ",dump($data);
+               warn "INFO imported $imported files from ",dump($data);
+
+               return; # don't put md5sum files into pool
        }
 
        if ( $data->{md5} ne $empty_md5 ) {
-               $self->md5sum($data)->put( $data->{file} => $data->{md5} );
                $self->md5pool( $data );
        } else {
                warn "empty md5", dump $data;