1 package CloudStore::Store;
7 use File::Path qw(make_path);
9 use Digest::MD5 qw(md5_base64);
10 use Data::Dump qw(dump);
22 die "no dir" unless $self->{dir};
23 $self->{md5pool} = $self->{dir} . '/md5';
25 warn "# new ",dump $self if $ENV{DEBUG};
31 my ( $self,$data ) = @_;
35 my ( $self,$data ) = @_;
40 $dir =~ s{/[^/]+$}{}; # strip filename
41 mkdir $dir unless -e $dir;
45 my ( $self,$data ) = @_;
47 if ( $data->{file} =~ m{^(.*/)?.sync/send/([^/]+)$} ) {
49 warn "SEND $2 from $from_dir\n";
51 open(my $send, '<', $self->blob_path($data) );
55 my ( $to, $file ) = split(/\s+/,$_,2);
56 my ( undef, undef, $uid, $gid, undef, undef, $email, $dir, $shell ) =
60 $from->{file} = $from_dir . $file;
61 my $from_path = $self->blob_path($from);
63 if ( ! -r $from_path ) {
64 warn "ERROR: $from_path: $!";
68 my $to_path = "$dir/received/$file";
71 warn "SEND $from_path -> $to_path\n";
72 unlink $to_path if -e $to_path; # FIXME why we need this?
73 $sent_files->{$to} += link $from_path, $to_path;
75 my $md5 = $self->md5sum($data)->get( $from_dir . $file ) || warn "no md5 for $from_dir$file";
76 $self->md5sum({login => $to})->put( "/received/$file" => $md5 );
79 warn "SENT ",dump $sent_files;
81 return 0; # skip dedup
82 } elsif ( $data->{file} =~ m{^(.*/)?.sync/pending/([^/]+)$} ) {
84 warn "PENDIG $2 from $from_dir";
85 open(my $pend, '<', $self->blob_path($data) );
89 if ( m/^DELETED\#(.+)$/ ) {
90 my $path = $from_dir . $1;
92 unlink $path && warn "unlink $path\n";
94 warn "MISSING $path to unlink\n";
96 } elsif ( ! /^(MOVED|RENAMED)\#/ ) {
101 my ( undef, $from, $to ) = split(/\#/,$_,3);
103 my ( $from_path, $to_path ) = map {
105 $tmp->{file} = $from_dir . $_;
106 $self->blob_path($tmp);
109 if ( ! -e $from_path ) {
110 warn "SKIPPED $from_path: $!";
114 warn "MV $from_path -> $to_path";
116 rename $from_path, $to_path;
118 my $md5 = $self->md5sum($data)->get( $from_dir . $from );
120 warn "ERROR: no md5sum $from_dir $from " unless $md5;
124 $self->md5sum($data)->out( $from_dir . $from );
125 $self->md5sum($data)->put( $from_dir . $to => $md5 );
127 warn "$md5 moved to $from_dir $to";
130 return 0; # skip dedup
133 #return $file->{size} > 4096 ? 1 : 0; # FIXME
137 # never called by rsync directly!
139 my ( $self,$data ) = @_;
140 # $self->file_set($data);
144 my ( $self, $data ) = @_;
146 my $md5 = $self->md5sum($data)->get( $data->{file} );
147 return unless $md5; # directories don't have md5sums
148 my $path = $self->{md5pool} . '/' . $md5;
149 my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
150 $atime,$mtime,$ctime,$blksize,$blocks)
153 my $id = getpwnam 'md5';
154 chown $id,$gid, $path;
155 warn "# chown $id $gid $path";
158 $self->md5sum($data)->out( $data->{file} );
162 my ( $self, $data ) = @_;
167 my ( $self, $data ) = @_;
169 warn "# new_link ",dump $data;
171 if ( $data->{file} =~ m{^(.*/?)\.send/([^/]+)/(.+)$} ) {
172 my ( $dir, $to, $name ) = ( $1, $2, $3 );
173 my $path = $self->blob_path($data);
174 my $link_to = readlink $path;
176 if ( $link_to =~ s{^\Q/rsyncd-munged/\E}{/} ) {
179 $s =~ s{/[^/]+$}{}; # strip filename
180 while ( $link_to =~ s{/../}{/} ) {
181 $s =~ s{/[^/]+$}{} || die "can't strip $s";
182 warn "## simplify $s $link_to\n";
186 my $d = $self->blob_path({
191 # $name can contain directories so we must create them
193 $to_dir =~ s{/[^/]+$}{};
194 make_path $to_dir if ! -e $to_dir;
197 warn "ERROR: can't find source $s";
200 warn "link $s -> $d\n";
203 my ($l,$f) = ($1,$2) if $s =~ m{users/([^/]+)/blob/(.+)};
205 # my $origin = $self->file_get({
209 # $self->new_file($origin);
210 warn "INFO: sent file ",dump($l,$f);
211 my $md5 = $self->md5sum($data)->get($s);
212 $self->md5sum({ login => $to })->put($d => $md5 );
218 warn "ERROR: can't SEND To:$to Name:$name Link:$link_to";
224 my ( $self, $data ) = @_;
226 my $login = $data->{login} || confess "missing login in ",dump $data;
228 return $self->{md5sum}->{$login} if exists $self->{md5sum}->{$login};
230 my ( undef, undef, $uid, $gid, undef, undef, $email, $dir, $shell ) =
233 my $md5_path = "$dir/.md5";
235 my $db = TokyoCabinet::HDB->new();
236 $db->open($md5_path, $db->OWRITER | $db->OCREAT)
237 or die "can't open $md5_path: ",$db->errmsg( $db->ecode );
239 warn "open $md5_path";
241 $self->{md5sum}->{$login} = $db;
246 my ( $self, $data ) = @_;
247 if ( $data =~ m/\[(\d+)\] rsync \w+ (\S+) \w+ (\S+)/ ) {
248 my ( $pid, $module, $login ) = ( $1, $2, $3 );
251 my ( undef, undef, $uid, $gid, undef, undef, $email, $dir, $shell ) =
254 $self->{pid}->{$pid} = {
265 } elsif ( $data =~ m/\[(\d+)\] sent \S+ bytes\s+received \S+ bytes/ ) {
268 foreach my $login ( keys %{ $self->{md5sum} } ) {
269 $self->{md5sum}->{$login}->close;
270 warn "close md5sum $login";
272 delete $self->{md5sum};
274 delete $self->{pid}->{$pid};
279 # warn "## rsync_log $data";
284 my ( $self, $data ) = @_;
285 my $blob = $self->{pid}->{ $data->{pid} }->{dir} || die "no dir for $data->{pid} in ",dump( $self->{pid} );
286 $blob .= '/' . $data->{file};
292 my ( $self,$data ) = @_;
294 my $path = $self->blob_path($data);
296 if ( $data->{itemize} =~ m/^[c>]([fdL])/ ) { # received change/create
299 if ( $type eq 'f' ) {
300 $self->modify_file( $data ) && # selective dedup
301 $self->dedup( $data, $path );
302 } elsif ( $type eq 'd' ) {
303 $self->make_dir( $data );
304 } elsif ( $type eq 'L' ) {
305 $self->new_link( $data );
307 die "unknown type $type ", dump $data;
309 } elsif ( $data->{itemize} =~ m/\*deleting/ ) {
310 $self->remove_file($data);
312 warn "IGNORED ",dump($data) if $ENV{DEBUG};
318 my ( $self, $data ) = @_;
320 my $pool = $self->{md5pool} || die "no md5pool in ",dump $self;
321 mkdir $pool unless -e $pool;
323 my $md5 = $data->{md5} || die "no md5 in ",dump $data;
324 my $path = $self->blob_path($data);
326 if ( -e "$pool/$md5" ) {
327 warn "dedup hit $md5 $path\n";
328 my $dedup = $path . '.dedup';
329 rename $path, $dedup;
330 link "$pool/$md5", $path;
334 link $path, "$pool/$md5";
335 warn "dedup +++ $md5 $path";
338 $self->md5sum($data)->put( $data->{file} => $md5 );
341 my $empty_md5 = " " x 32;
344 my ( $self, $data, $path ) = @_;
346 if ( $data->{file} =~ /^(.+\/)?md5sum$/ ) {
349 warn "IMPORT ", $data->{file}, "\n";
350 open(my $md5sum, '<', $path);
353 my ( $md5, $file ) = split(/\s+/,$_,2);
354 if ( ! -e "$self->{md5path}/$md5" ) {
355 warn "MISSING $md5 $file\n";
363 my $new_path = $self->blob_path($new);
364 if ( ! -e $new_path ) {
365 # create path from md5sum file
366 my $only_dir = $1 if $new =~ m{^(.+)/[^/]+$};
367 make_path $only_dir unless -d $only_dir;
368 $imported += link "$self->{md5path}/$md5", $new_path;
369 $self->new_file($new);
370 warn "import from $path ",dump($new);
371 $self->md5pool( $new );
373 $self->md5pool( $new );
376 print "INFO imported $imported files from ",dump($data);
379 if ( $data->{md5} ne $empty_md5 ) {
380 $self->md5pool( $data );
382 warn "empty md5", dump $data;