1 package CloudStore::Store;
6 use base 'CloudStore::MD5sum';
11 use File::Path qw(make_path);
13 use Digest::MD5 qw(md5_base64);
14 use Data::Dump qw(dump);
20 my ($class,$group) = @_;
23 api => CloudStore::API->new( $group ),
27 $self->{md5pool} = $self->{api}->{md5}->{dir};
29 warn "# new ",dump $self if $ENV{DEBUG};
36 $dir =~ s{/[^/]+$}{}; # strip filename
37 mkdir $dir unless -e $dir;
41 my ( $self,$data ) = @_;
45 if ( $data->{file} =~ m{^(.*/)?.sync/send/([^/]+)$} ) {
47 warn "SEND $2 from $from_dir\n";
49 open(my $send, '<', $self->blob_path($data) );
53 my ( $to, $file ) = split(/\s+/,$_,2);
54 my ( undef, undef, $uid, $gid, undef, undef, $email, $dir, $shell ) =
58 $from->{file} = $from_dir . $file;
59 my $from_path = $self->blob_path($from);
61 if ( ! -r $from_path ) {
62 warn "ERROR: $from_path: $!";
66 my $to_path = "$dir/received/$file";
69 warn "SEND $from_path -> $to_path\n";
70 unlink $to_path if -e $to_path; # FIXME why we need this?
71 $sent_files->{$to} += link $from_path, $to_path;
73 my $md5 = $self->md5sum($data)->get( $from_dir . $file ) || warn "no md5 for $from_dir$file";
74 $self->md5sum({login => $to})->put( "/received/$file" => $md5 );
77 warn "SENT ",dump $sent_files;
79 return 0; # skip dedup
80 } elsif ( $data->{file} =~ m{^(.*/)?.sync/pending/([^/]+)$} ) {
82 warn "PENDIG $2 from $from_dir";
83 open(my $pend, '<', $self->blob_path($data) );
87 if ( m/^DELETED\#(.+)$/ ) {
88 my $path = $self->blob_path($data => $from_dir . $1 );
91 -d $path ? rmdir $path : unlink $path || warn "ERROR: unlink $path $!";
94 warn "MISSING $path to unlink";
97 } elsif ( ! /^(MOVED|RENAMED)\#/ ) {
102 my ( undef, $from, $to ) = split(/\#/,$_,3);
104 my ( $from_path, $to_path ) = map {
106 $tmp->{file} = $from_dir . $_;
107 $self->blob_path($tmp);
110 if ( ! -e $from_path ) {
111 warn "SKIPPED $from_path: $!";
115 warn "MV $from_path -> $to_path";
117 rename $from_path, $to_path;
119 my $md5 = $self->md5sum($data)->get( $from_dir . $from );
121 warn "ERROR: no md5sum $from_dir $from " unless $md5;
125 $self->md5sum($data)->out( $from_dir . $from );
126 $self->md5sum($data)->put( $from_dir . $to => $md5 );
128 warn "$md5 moved to $from_dir $to";
131 return 0; # skip dedup
136 if ( $data->{file} =~ m{^(.*/)?.sync/} ) {
137 # ignore .sync/ files from client
141 #return $file->{size} > 4096 ? 1 : 0; # FIXME
145 # never called by rsync directly!
147 my ( $self,$data ) = @_;
148 # $self->file_set($data);
151 # client doesn't issue --delete
153 my ( $self, $data ) = @_;
155 my $md5 = $self->md5sum($data)->get( $data->{file} );
156 return unless $md5; # directories don't have md5sums
157 my $path = $self->{md5pool} . '/' . $md5;
158 my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
159 $atime,$mtime,$ctime,$blksize,$blocks)
162 my $user = $self->{api}->user_info($data->{login});
164 if ( $nlink == 1 && $uid == $user->{uid} ) {
165 $self->append( $user, 'removed', -$size, $uid, $data->{file} );
166 my $id = getpwnam 'md5';
167 chown $id,$gid, $path;
168 warn "# chown $id $gid $path";
171 $self->md5sum($data)->out( $data->{file} );
175 my ( $self, $data ) = @_;
180 my ( $self, $data ) = @_;
182 warn "# new_link ",dump $data;
186 if ( $data->{file} =~ m{^(.*/?)\.send/([^/]+)/(.+)$} ) {
187 my ( $dir, $to, $name ) = ( $1, $2, $3 );
188 my $path = $self->blob_path($data);
189 my $link_to = readlink $path;
191 if ( $link_to =~ s{^\Q/rsyncd-munged/\E}{/} ) {
194 $s =~ s{/[^/]+$}{}; # strip filename
195 while ( $link_to =~ s{/../}{/} ) {
196 $s =~ s{/[^/]+$}{} || die "can't strip $s";
197 warn "## simplify $s $link_to\n";
201 my $d = $self->blob_path({
206 # $name can contain directories so we must create them
210 warn "ERROR: can't find source $s";
213 warn "link $s -> $d\n";
216 my ($l,$f) = ($1,$2) if $s =~ m{users/([^/]+)/blob/(.+)};
218 # my $origin = $self->file_get({
222 # $self->new_file($origin);
223 warn "INFO: sent file ",dump($l,$f);
224 my $md5 = $self->md5sum($data)->get($s);
225 $self->md5sum({ login => $to })->put($d => $md5 );
231 warn "ERROR: can't SEND To:$to Name:$name Link:$link_to";
240 my ( $self, $pid, $login ) = @_;
243 $self->{pid}->{$pid} = $self->{api}->user_info($login);
249 my ( $self, $pid ) = @_;
253 delete $self->{pid}->{$pid};
258 my ( $self, $data ) = @_;
259 if ( $data =~ m/\[(\d+)\] rsync \w+ (\S+) \w+ (\S+)/ ) {
260 my ( $pid, $module, $login ) = ( $1, $2, $3 );
261 $self->init_pid_login( $pid, $login );
262 } elsif ( $data =~ m/\[(\d+)\] sent \S+ bytes\s+received \S+ bytes/ ) {
264 $self->cleanup_pid( $pid );
266 warn "## rsync_log $data";
271 my ( $self, $data, $path ) = @_;
272 my $blob = $self->{pid}->{ $data->{pid} }->{dir} || die "no dir for $data->{pid} in ",dump( $self->{pid} );
273 $blob .= '/' . ( defined $path ? $path : $data->{file} );
279 my ( $self,$data ) = @_;
281 my $path = $self->blob_path($data);
283 if ( $data->{itemize} =~ m/^[c>]([fdL])/ ) { # received change/create
286 if ( $type eq 'f' ) {
287 $self->modify_file( $data ) && # selective dedup
288 $self->dedup( $data, $path );
289 } elsif ( $type eq 'd' ) {
290 $self->make_dir( $data );
291 } elsif ( $type eq 'L' ) {
292 $self->new_link( $data );
294 die "unknown type $type ", dump $data;
296 } elsif ( $data->{itemize} =~ m/\*deleting/ ) {
297 $self->removed_file($data);
299 warn "IGNORED ",dump($data) if $ENV{DEBUG};
306 $self->{api}->append( @_ );
310 my ( $self, $data ) = @_;
312 my $pool = $self->{md5pool} || die "no md5pool in ",dump $self;
313 mkdir $pool unless -e $pool;
315 my $md5 = $data->{md5} || die "no md5 in ",dump $data;
316 my $path = $self->blob_path($data);
318 my $pool_md5 = "$pool/$md5";
320 if ( -e $pool_md5 ) {
321 warn "dedup hit $md5 $path\n";
323 my ($pool_uid,$pool_size) = (stat($pool_md5))[4,7];
324 my $user = $self->{api}->user_info( $data->{login} );
326 if ( $pool_uid != $self->{api}->{md5}->{uid} ) {
327 chown $self->{api}->{md5}->{uid}, $self->{api}->{md5}->{gid}, $pool_md5;
328 chmod oct("0444"), $pool_md5;
329 my $steal_user = $self->{api}->user_info( $pool_uid );
330 $self->append( $steal_user, 'dedup-steal', $pool_size, $pool_uid, $data->{file} );
332 $self->append( $user, 'dedup', $pool_size, $pool_uid, $data->{file} );
334 my $dedup = $path . '.dedup';
335 rename $path, $dedup;
336 link "$pool/$md5", $path;
339 link $path, "$pool/$md5";
340 warn "dedup +++ $md5 $path";
343 $self->md5sum($data)->put( $data->{file} => $md5 );
346 my $empty_md5 = " " x 32;
349 my ( $self, $data, $path ) = @_;
351 if ( $data->{file} =~ /^(.+\/)?md5sum$/ ) {
354 warn "IMPORT ", $data->{file}, "\n";
355 open(my $md5sum, '<', $path);
358 my ( $md5, $file ) = split(/\s+/,$_,2);
359 if ( ! -e "$self->{md5path}/$md5" ) {
360 warn "MISSING $md5 $file\n";
368 my $new_path = $self->blob_path($new);
369 if ( ! -e $new_path ) {
370 $self->{api}->send_file( 'md5' => $md5, $data->{login}, "$dir$file" );
371 $self->md5pool( $new );
373 $self->md5pool( $new );
376 print "INFO imported $imported files from ",dump($data);
379 if ( $data->{md5} ne $empty_md5 ) {
380 $self->md5sum($data)->put( $data->{file} => $data->{md5} );
381 $self->md5pool( $data );
383 warn "empty md5", dump $data;