1 #============================================================= -*-perl-*-
8 # Craig Barratt <cbarratt@users.sourceforge.net>
11 # Copyright (C) 2002-2009 Craig Barratt
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation; either version 2 of the License, or
16 # (at your option) any later version.
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
23 # You should have received a copy of the GNU General Public License
24 # along with this program; if not, write to the Free Software
25 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27 #========================================================================
29 # Version 3.2.0, released 31 Jul 2010.
31 # See http://backuppc.sourceforge.net.
33 #========================================================================
35 package BackupPC::Xfer::RsyncFileIO;
39 use Encode qw/from_to/;
40 use BackupPC::Attrib qw(:all);
42 use BackupPC::Xfer::RsyncDigest qw(:all);
43 use BackupPC::PoolWrite;
45 use constant S_HLINK_TARGET => 0400000; # this file is hardlink target
46 use constant S_IFMT => 0170000; # type of file
47 use constant S_IFDIR => 0040000; # directory
48 use constant S_IFCHR => 0020000; # character special
49 use constant S_IFBLK => 0060000; # block special
50 use constant S_IFREG => 0100000; # regular
51 use constant S_IFLNK => 0120000; # symbolic link
52 use constant S_IFSOCK => 0140000; # socket
53 use constant S_IFIFO => 0010000; # fifo
55 use vars qw( $RsyncLibOK );
58 eval "use File::RsyncP::Digest";
61 # Rsync module doesn't exist.
71 my($class, $options) = @_;
73 return if ( !$RsyncLibOK );
78 digest => File::RsyncP::Digest->new(),
81 logHandler => \&logHandler,
88 ExistFileCompSize => 0,
93 $fio->{digest}->protocol($fio->{protocol_version});
94 $fio->{shareM} = $fio->{bpc}->fileNameEltMangle($fio->{share});
95 $fio->{outDir} = "$fio->{xfer}{outDir}/new/";
96 $fio->{outDirSh} = "$fio->{outDir}/$fio->{shareM}/";
97 $fio->{view} = BackupPC::View->new($fio->{bpc}, $fio->{client},
99 $fio->{full} = $fio->{xfer}{type} eq "full" ? 1 : 0;
100 $fio->{newFilesFH} = $fio->{xfer}{newFilesFH};
101 $fio->{partialNum} = undef if ( !$fio->{full} );
106 # We publish our version to File::RsyncP. This is so File::RsyncP
107 # can provide backward compatibility to older FileIO code.
111 # undef or 1: protocol version 26, no hardlinks
112 # 2: protocol version 28, supports hardlinks
121 my($fio, $value) = @_;
123 $fio->{blockSize} = $value if ( defined($value) );
124 return $fio->{blockSize};
129 my($fio, $value) = @_;
131 if ( defined($value) ) {
132 $fio->{protocol_version} = $value;
133 $fio->{digest}->protocol($fio->{protocol_version});
135 return $fio->{protocol_version};
138 sub preserve_hard_links
140 my($fio, $value) = @_;
142 $fio->{preserve_hard_links} = $value if ( defined($value) );
143 return $fio->{preserve_hard_links};
149 $fio->{logHandler} = $sub;
150 BackupPC::Xfer::RsyncDigest->logHandlerSet($sub);
154 # Setup rsync checksum computation for the given file.
158 my($fio, $f, $needMD4, $defBlkSize, $phase) = @_;
160 $defBlkSize ||= $fio->{blockSize};
161 my $attr = $fio->attribGet($f, 1);
163 $fio->csumEnd if ( defined($fio->{csum}) );
164 return -1 if ( $attr->{type} != BPC_FTYPE_FILE );
167 # Rsync uses short checksums on the first phase. If the whole-file
168 # checksum fails, then the file is repeated with full checksums.
169 # So on phase 2 we verify the checksums if they are cached.
171 if ( ($phase > 0 || rand(1) < $fio->{cacheCheckProb})
173 && $fio->{checksumSeed} == RSYNC_CSUMSEED_CACHE ) {
174 my($err, $d, $blkSize) = BackupPC::Xfer::RsyncDigest->digestStart(
175 $attr->{fullPath}, $attr->{size}, 0,
176 $defBlkSize, $fio->{checksumSeed},
177 0, $attr->{compress}, 0,
178 $fio->{protocol_version});
180 $fio->log("Can't get rsync digests from $attr->{fullPath}"
181 . " (err=$err, name=$f->{name})");
182 $fio->{stats}{errorCnt}++;
185 my($isCached, $isInvalid) = $d->isCached;
186 if ( $fio->{logLevel} >= 5 ) {
187 $fio->log("$attr->{fullPath} verify; cached = $isCached,"
188 . " invalid = $isInvalid, phase = $phase");
190 if ( $isCached || $isInvalid ) {
191 my $ret = BackupPC::Xfer::RsyncDigest->digestAdd(
192 $attr->{fullPath}, $blkSize,
193 $fio->{checksumSeed}, 1, # verify
194 $fio->{protocol_version}
197 $fio->log("Bad cached digest for $attr->{fullPath} ($ret);"
199 $fio->{stats}{errorCnt}++;
201 $fio->log("$f->{name}: verified cached digest")
202 if ( $fio->{logLevel} >= 2 );
207 (my $err, $fio->{csum}, my $blkSize)
208 = BackupPC::Xfer::RsyncDigest->digestStart($attr->{fullPath},
209 $attr->{size}, 0, $defBlkSize, $fio->{checksumSeed},
210 $needMD4, $attr->{compress}, 1, $fio->{protocol_version});
212 $fio->log("Can't get rsync digests from $attr->{fullPath}"
213 . " (err=$err, name=$f->{name})");
214 $fio->{stats}{errorCnt}++;
217 if ( $fio->{logLevel} >= 5 ) {
218 my($isCached, $invalid) = $fio->{csum}->isCached;
219 $fio->log("$attr->{fullPath} cache = $isCached,"
220 . " invalid = $invalid, phase = $phase");
227 my($fio, $num, $csumLen, $blockSize) = @_;
232 return if ( !defined($fio->{csum}) );
233 return $fio->{csum}->digestGet($num, $csumLen);
240 return if ( !defined($fio->{csum}) );
241 return $fio->{csum}->digestEnd();
248 my $attr = $fio->attribGet($f, 1);
250 $fio->readEnd if ( defined($fio->{fh}) );
251 if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath},
253 $attr->{compress})) ) {
254 $fio->log("Can't open $attr->{fullPath} (name=$f->{name})");
255 $fio->{stats}{errorCnt}++;
258 $fio->log("$f->{name}: opened for read") if ( $fio->{logLevel} >= 4 );
267 return if ( !defined($fio->{fh}) );
268 if ( $fio->{fh}->read(\$fileData, $num) <= 0 ) {
269 return $fio->readEnd;
271 $fio->log(sprintf("read returns %d bytes", length($fileData)))
272 if ( $fio->{logLevel} >= 8 );
280 return if ( !defined($fio->{fh}) );
282 $fio->log("closing $fio->{file}{name})") if ( $fio->{logLevel} >= 8 );
289 my($fio, $checksumSeed) = @_;
291 $fio->{checksumSeed} = $checksumSeed;
292 $fio->log("Checksum caching enabled (checksumSeed = $checksumSeed)")
293 if ( $fio->{logLevel} >= 1 && $checksumSeed == RSYNC_CSUMSEED_CACHE );
294 $fio->log("Checksum seed is $checksumSeed")
295 if ( $fio->{logLevel} >= 2 && $checksumSeed != RSYNC_CSUMSEED_CACHE );
300 my($fio, $localDir, $remoteDir) = @_;
302 $fio->{localDir} = $localDir;
303 $fio->{remoteDir} = $remoteDir;
308 my($fio, $share, $dir) = @_;
311 #$fio->log("viewCacheDir($share, $dir)");
312 if ( !defined($share) ) {
313 $share = $fio->{share};
314 $shareM = $fio->{shareM};
316 $shareM = $fio->{bpc}->fileNameEltMangle($share);
318 $shareM = "$shareM/$dir" if ( $dir ne "" );
319 return if ( defined($fio->{viewCache}{$shareM}) );
321 # purge old cache entries (ie: those that don't match the
322 # first part of $dir).
324 foreach my $d ( keys(%{$fio->{viewCache}}) ) {
325 delete($fio->{viewCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
328 # fetch new directory attributes
330 $fio->{viewCache}{$shareM}
331 = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir);
333 # also cache partial backup attrib data too
335 if ( defined($fio->{partialNum}) ) {
336 foreach my $d ( keys(%{$fio->{partialCache}}) ) {
337 delete($fio->{partialCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
339 $fio->{partialCache}{$shareM}
340 = $fio->{view}->dirAttrib($fio->{partialNum}, $share, $dir);
346 my($fio, $f, $noCache, $fname) = @_;
347 my($dir, $share, $shareM, $partial, $attr);
349 if ( !defined($fname) ) {
351 $fname = "$fio->{xfer}{pathHdrSrc}/$fname"
352 if ( defined($fio->{xfer}{pathHdrSrc}) );
354 $fname =~ s{//+}{/}g;
355 if ( $fname =~ m{(.*)/(.*)}s ) {
356 $shareM = $fio->{shareM};
359 } elsif ( $fname ne "." ) {
360 $shareM = $fio->{shareM};
366 $fname = $fio->{share};
368 $shareM .= "/$dir" if ( $dir ne "" );
371 $share = $fio->{share} if ( !defined($share) );
372 my $dirAttr = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir);
373 $attr = $dirAttr->{$fname};
375 $fio->viewCacheDir($share, $dir);
376 if ( defined($attr = $fio->{viewCache}{$shareM}{$fname}) ) {
378 } elsif ( defined($attr = $fio->{partialCache}{$shareM}{$fname}) ) {
383 if ( $attr->{mode} & S_HLINK_TARGET ) {
384 $attr->{hlink_self} = 1;
385 $attr->{mode} &= ~S_HLINK_TARGET;
388 return ($attr, $partial);
393 my($fio, $f, $doHardLink) = @_;
395 my($attr) = $fio->attribGetWhere($f);
396 if ( $doHardLink && $attr->{type} == BPC_FTYPE_HARDLINK ) {
397 $fio->log("$attr->{fullPath}: opening for hardlink read"
398 . " (name = $f->{name})") if ( $fio->{logLevel} >= 4 );
399 my $fh = BackupPC::FileZIO->open($attr->{fullPath}, 0,
402 if ( defined($fh) ) {
403 $fh->read(\$target, 65536);
405 $target =~ s/^\.?\/+//;
407 $fio->log("$attr->{fullPath}: can't open for hardlink read");
408 $fio->{stats}{errorCnt}++;
409 $attr->{type} = BPC_FTYPE_FILE;
412 $target = "/$target" if ( $target !~ /^\// );
413 $fio->log("$attr->{fullPath}: redirecting to $target")
414 if ( $fio->{logLevel} >= 4 );
416 ($attr) = $fio->attribGetWhere($f, 1, $target);
417 $fio->log(" ... now got $attr->{fullPath}")
418 if ( $fio->{logLevel} >= 4 );
426 my $mode = $f->{mode};
428 if ( ($mode & S_IFMT) == S_IFREG ) {
429 if ( defined($f->{hlink}) && !$f->{hlink_self} ) {
430 return BPC_FTYPE_HARDLINK;
432 return BPC_FTYPE_FILE;
434 } elsif ( ($mode & S_IFMT) == S_IFDIR ) {
435 return BPC_FTYPE_DIR;
436 } elsif ( ($mode & S_IFMT) == S_IFLNK ) {
437 return BPC_FTYPE_SYMLINK;
438 } elsif ( ($mode & S_IFMT) == S_IFCHR ) {
439 return BPC_FTYPE_CHARDEV;
440 } elsif ( ($mode & S_IFMT) == S_IFBLK ) {
441 return BPC_FTYPE_BLOCKDEV;
442 } elsif ( ($mode & S_IFMT) == S_IFIFO ) {
443 return BPC_FTYPE_FIFO;
444 } elsif ( ($mode & S_IFMT) == S_IFSOCK ) {
445 return BPC_FTYPE_SOCKET;
447 return BPC_FTYPE_UNKNOWN;
452 # Set the attributes for a file. Returns non-zero on error.
456 my($fio, $f, $placeHolder) = @_;
459 return if ( $placeHolder && $fio->{phase} > 0 );
461 if ( $f->{name} =~ m{(.*)/(.*)}s ) {
463 $dir = "$fio->{shareM}/" . $1;
464 } elsif ( $f->{name} eq "." ) {
466 $file = $fio->{share};
468 $dir = $fio->{shareM};
473 && (!defined($fio->{attribLastDir}) || $fio->{attribLastDir} ne $dir) ) {
475 # Flush any directories that don't match the first part
476 # of the new directory. Don't flush the top-level directory
477 # (ie: $dir eq "") since the "." might get sorted in the middle
478 # of other top-level directories or files.
480 foreach my $d ( keys(%{$fio->{attrib}}) ) {
481 next if ( $d eq "" || "$dir/" =~ m{^\Q$d/} );
482 $fio->attribWrite($d);
484 $fio->{attribLastDir} = $dir;
486 if ( !exists($fio->{attrib}{$dir}) ) {
487 $fio->log("attribSet: dir=$dir not found") if ( $fio->{logLevel} >= 4 );
488 $fio->{attrib}{$dir} = BackupPC::Attrib->new({
489 compress => $fio->{xfer}{compress},
492 $dirM = $1 . "/" . $fio->{bpc}->fileNameMangle($2)
493 if ( $dirM =~ m{(.*?)/(.*)}s );
494 my $path = $fio->{outDir} . $dirM;
495 if ( -f $fio->{attrib}{$dir}->fileName($path) ) {
496 if ( !$fio->{attrib}{$dir}->read($path) ) {
497 $fio->log(sprintf("Unable to read attribute file %s",
498 $fio->{attrib}{$dir}->fileName($path)));
500 $fio->log(sprintf("attribRead file %s",
501 $fio->{attrib}{$dir}->fileName($path)))
502 if ( $fio->{logLevel} >= 4 );
506 $fio->log("attribSet: dir=$dir exists") if ( $fio->{logLevel} >= 4 );
508 $fio->log("attribSet(dir=$dir, file=$file, size=$f->{size}, placeholder=$placeHolder)")
509 if ( $fio->{logLevel} >= 4 );
511 my $mode = $f->{mode};
513 $mode |= S_HLINK_TARGET if ( $f->{hlink_self} );
514 $fio->{attrib}{$dir}->set($file, {
515 type => $fio->mode2type($f),
519 size => $placeHolder ? -1 : $f->{size},
520 mtime => $f->{mtime},
530 if ( !defined($d) ) {
532 # flush all entries (in reverse order)
534 foreach $d ( sort({$b cmp $a} keys(%{$fio->{attrib}})) ) {
535 $fio->attribWrite($d);
539 return if ( !defined($fio->{attrib}{$d}) );
542 # Set deleted files in the attributes. Any file in the view
543 # that doesn't have attributes is flagged as deleted for
544 # incremental dumps. All files sent by rsync have attributes
545 # temporarily set so we can do deletion detection. We also
546 # prune these temporary attributes.
552 $dir = $1 if ( $d =~ m{.+?/(.*)}s );
553 $fio->viewCacheDir(undef, $dir);
554 ##print("attribWrite $d,$dir\n");
555 ##$Data::Dumper::Indent = 1;
556 ##$fio->log("attribWrite $d,$dir");
557 ##$fio->log("viewCacheLogKeys = ", keys(%{$fio->{viewCache}}));
558 ##$fio->log("attribKeys = ", keys(%{$fio->{attrib}}));
559 ##print "viewCache = ", Dumper($fio->{attrib});
560 ##print "attrib = ", Dumper($fio->{attrib});
561 if ( defined($fio->{viewCache}{$d}) ) {
562 foreach my $f ( keys(%{$fio->{viewCache}{$d}}) ) {
564 $name = "$1/$name" if ( $d =~ m{.*?/(.*)}s );
565 if ( defined(my $a = $fio->{attrib}{$d}->get($f)) ) {
567 # delete temporary attributes (skipped files)
569 if ( $a->{size} < 0 ) {
570 $fio->{attrib}{$d}->set($f, undef);
571 $fio->logFileAction("skip", {
572 %{$fio->{viewCache}{$d}{$f}},
574 }) if ( $fio->{logLevel} >= 2
575 && $a->{type} == BPC_FTYPE_FILE );
577 } elsif ( $fio->{phase} == 0 && !$fio->{full} ) {
578 ##print("Delete file $f\n");
579 $fio->logFileAction("delete", {
580 %{$fio->{viewCache}{$d}{$f}},
582 }) if ( $fio->{logLevel} >= 1 );
583 $fio->{attrib}{$d}->set($f, {
584 type => BPC_FTYPE_DELETED,
595 if ( $fio->{attrib}{$d}->fileCount || $fio->{phase} > 0 ) {
596 my $data = $fio->{attrib}{$d}->writeData;
599 $dirM = $1 . "/" . $fio->{bpc}->fileNameMangle($2)
600 if ( $dirM =~ m{(.*?)/(.*)}s );
601 my $fileName = $fio->{attrib}{$d}->fileName("$fio->{outDir}$dirM");
602 $fio->log("attribWrite(dir=$d) -> $fileName")
603 if ( $fio->{logLevel} >= 4 );
604 my $poolWrite = BackupPC::PoolWrite->new($fio->{bpc}, $fileName,
605 length($data), $fio->{xfer}{compress});
606 $poolWrite->write(\$data);
607 $fio->processClose($poolWrite, $fio->{attrib}{$d}->fileName($dirM),
610 delete($fio->{attrib}{$d});
615 my($fio, $poolWrite, $fileName, $origSize, $doStats) = @_;
616 my($exists, $digest, $outSize, $errs) = $poolWrite->close;
618 $fileName =~ s{^/+}{};
619 if ( defined($errs) && @$errs ) {
621 $fio->{stats}{errorCnt} += @$errs;
624 $fio->{stats}{TotalFileCnt}++;
625 $fio->{stats}{TotalFileSize} += $origSize;
629 $fio->{stats}{ExistFileCnt}++;
630 $fio->{stats}{ExistFileSize} += $origSize;
631 $fio->{stats}{ExistFileCompSize} += $outSize;
633 } elsif ( $outSize > 0 ) {
634 my $fh = $fio->{newFilesFH};
635 print($fh "$digest $origSize $fileName\n") if ( defined($fh) );
637 return $exists && $origSize > 0;
644 return $fio->{stats};
648 # Make a given directory. Returns non-zero on error.
653 my $name = $1 if ( $f->{name} =~ /(.*)/s );
656 if ( $name eq "." ) {
657 $path = $fio->{outDirSh};
659 $path = $fio->{outDirSh} . $fio->{bpc}->fileNameMangle($name);
661 $fio->logFileAction("create", $f) if ( $fio->{logLevel} >= 1 );
662 $fio->log("makePath($path, 0777)") if ( $fio->{logLevel} >= 5 );
663 $path = $1 if ( $path =~ /(.*)/s );
664 eval { File::Path::mkpath($path, 0, 0777) } if ( !-d $path );
665 return $fio->attribSet($f) if ( -d $path );
666 $fio->log("Can't create directory $path");
667 $fio->{stats}{errorCnt}++;
672 # Make a special file. Returns non-zero on error.
677 my $name = $1 if ( $f->{name} =~ /(.*)/s );
678 my $fNameM = $fio->{bpc}->fileNameMangle($name);
679 my $path = $fio->{outDirSh} . $fNameM;
680 my $attr = $fio->attribGet($f);
682 my $type = $fio->mode2type($f);
684 $fio->log("makeSpecial($path, $type, $f->{mode})")
685 if ( $fio->{logLevel} >= 5 );
686 if ( $type == BPC_FTYPE_CHARDEV || $type == BPC_FTYPE_BLOCKDEV ) {
687 my($major, $minor, $fh, $fileData);
689 if ( defined($f->{rdev_major}) ) {
690 $major = $f->{rdev_major};
691 $minor = $f->{rdev_minor};
693 $major = $f->{rdev} >> 8;
694 $minor = $f->{rdev} & 0xff;
696 $str = "$major,$minor";
697 } elsif ( ($f->{mode} & S_IFMT) == S_IFLNK ) {
699 } elsif ( ($f->{mode} & S_IFMT) == S_IFREG ) {
703 if ( !defined($f->{hlink}) ) {
704 $fio->log("Error: makeSpecial($path, $type, $f->{mode}) called"
705 . " on a regular non-hardlink file");
711 # Now see if the file is different, or this is a full, in which
712 # case we create the new file.
717 || $attr->{type} != $type
718 || $attr->{mtime} != $f->{mtime}
719 || $attr->{size} != $f->{size}
720 || $attr->{uid} != $f->{uid}
721 || $attr->{gid} != $f->{gid}
722 || $attr->{mode} != $f->{mode}
723 || $attr->{hlink_self} != $f->{hlink_self}
724 || !defined($fh = BackupPC::FileZIO->open($attr->{fullPath}, 0,
726 || $fh->read(\$fileData, length($str) + 1) != length($str)
727 || $fileData ne $str ) {
728 $fh->close if ( defined($fh) );
729 $fh = BackupPC::PoolWrite->new($fio->{bpc}, $path,
730 length($str), $fio->{xfer}{compress});
732 my $exist = $fio->processClose($fh, "$fio->{shareM}/$fNameM",
734 $fio->logFileAction($exist ? "pool" : "create", $f)
735 if ( $fio->{logLevel} >= 1 );
736 return $fio->attribSet($f);
738 $fio->logFileAction("skip", $f) if ( $fio->{logLevel} >= 2 );
740 $fh->close if ( defined($fh) );
744 # Make a hardlink. Returns non-zero on error.
745 # This actually gets called twice for each hardlink.
746 # Once as the file list is processed, and again at
747 # the end. BackupPC does them as it goes (since it is
748 # just saving the hardlink info and not actually making
753 my($fio, $f, $end) = @_;
756 return $fio->makeSpecial($f) if ( !$f->{hlink_self} );
761 my($fio, $path) = @_;
763 $fio->log("Unexpected call BackupPC::Xfer::RsyncFileIO->unlink($path)");
767 # Default log handler
773 print(STDERR $str, "\n");
777 # Handle one or more log messages
781 my($fio, @logStr) = @_;
783 foreach my $str ( @logStr ) {
784 next if ( $str eq "" );
785 $fio->{logHandler}($str);
790 # Generate a log file message for a completed file
794 my($fio, $action, $f) = @_;
795 my $owner = "$f->{uid}/$f->{gid}";
796 my $type = (("", "p", "c", "", "d", "", "b", "", "", "", "l", "", "s"))
797 [($f->{mode} & S_IFMT) >> 12];
798 my $name = $f->{name};
800 if ( ($f->{mode} & S_IFMT) == S_IFLNK ) {
801 $name .= " -> $f->{link}";
802 } elsif ( ($f->{mode} & S_IFMT) == S_IFREG
803 && defined($f->{hlink}) && !$f->{hlink_self} ) {
804 $name .= " -> $f->{hlink}";
808 $fio->log(sprintf(" %-6s %1s%4o %9s %11.0f %s",
818 # If there is a partial and we are doing a full, we do an incremental
819 # against the partial and a full against the rest. This subroutine
820 # is how we tell File::RsyncP which files to ignore attributes on
821 # (ie: against the partial dump we do consider the attributes, but
822 # otherwise we ignore attributes).
828 return if ( !defined($fio->{partialNum}) );
829 my($attr, $isPartial) = $fio->attribGetWhere($f);
830 $fio->log("$f->{name}: just checking attributes from partial")
831 if ( $isPartial && $fio->{logLevel} >= 5 );
836 # This is called by File::RsyncP when a file is skipped because the
841 my($fio, $f, $attr) = @_;
844 # Unless this is a partial, this is normal so ignore it.
846 return if ( !defined($fio->{partialNum}) );
848 $fio->log("$f->{name}: skipped in partial; adding link")
849 if ( $fio->{logLevel} >= 5 );
850 $fio->{rxLocalAttr} = $attr;
852 $fio->{rxSize} = $attr->{size};
853 delete($fio->{rxInFd});
854 delete($fio->{rxOutFd});
855 delete($fio->{rxDigest});
856 delete($fio->{rxInData});
857 return $fio->fileDeltaRxDone();
861 # Start receive of file deltas for a particular file.
865 my($fio, $f, $cnt, $size, $remainder) = @_;
867 $fio->{rxFile} = $f; # remote file attributes
868 $fio->{rxLocalAttr} = $fio->attribGet($f); # local file attributes
869 $fio->{rxBlkCnt} = $cnt; # how many blocks we will receive
870 $fio->{rxBlkSize} = $size; # block size
871 $fio->{rxRemainder} = $remainder; # size of the last block
872 $fio->{rxMatchBlk} = 0; # current start of match
873 $fio->{rxMatchNext} = 0; # current next block of match
874 $fio->{rxSize} = 0; # size of received file
875 my $rxSize = $cnt > 0 ? ($cnt - 1) * $size + $remainder : 0;
876 if ( $fio->{rxFile}{size} != $rxSize ) {
877 $fio->{rxMatchBlk} = undef; # size different, so no file match
878 $fio->log("$fio->{rxFile}{name}: size doesn't match"
879 . " ($fio->{rxFile}{size} vs $rxSize)")
880 if ( $fio->{logLevel} >= 5 );
883 # If compression was off and now on, or on and now off, then
884 # don't do an exact match.
886 if ( defined($fio->{rxLocalAttr})
887 && !$fio->{rxLocalAttr}{compress} != !$fio->{xfer}{compress} ) {
888 $fio->{rxMatchBlk} = undef; # compression changed, so no file match
889 $fio->log("$fio->{rxFile}{name}: compression changed, so no match"
890 . " ($fio->{rxLocalAttr}{compress} vs $fio->{xfer}{compress})")
891 if ( $fio->{logLevel} >= 4 );
894 # If the local file is a hardlink then no match
896 if ( defined($fio->{rxLocalAttr})
897 && $fio->{rxLocalAttr}{type} == BPC_FTYPE_HARDLINK ) {
898 $fio->{rxMatchBlk} = undef;
899 $fio->log("$fio->{rxFile}{name}: no match on hardlinks")
900 if ( $fio->{logLevel} >= 4 );
902 # need to copy since hardlink attribGet overwrites the name
904 $fio->{rxHLinkAttr} = $fio->attribGet($fCopy, 1); # hardlink attributes
906 delete($fio->{rxHLinkAttr});
908 delete($fio->{rxInFd});
909 delete($fio->{rxOutFd});
910 delete($fio->{rxDigest});
911 delete($fio->{rxInData});
915 # Process the next file delta for the current file. Returns 0 if ok,
916 # -1 if not. Must be called with either a block number, $blk, or new data,
917 # $newData, (not both) defined.
921 my($fio, $blk, $newData) = @_;
923 if ( defined($blk) ) {
924 if ( defined($fio->{rxMatchBlk}) && $fio->{rxMatchNext} == $blk ) {
926 # got the next block in order; just keep track.
928 $fio->{rxMatchNext}++;
932 my $newDataLen = length($newData);
933 $fio->log("$fio->{rxFile}{name}: blk=$blk, newData=$newDataLen, rxMatchBlk=$fio->{rxMatchBlk}, rxMatchNext=$fio->{rxMatchNext}")
934 if ( $fio->{logLevel} >= 8 );
935 if ( !defined($fio->{rxOutFd}) ) {
937 # maybe the file has no changes
939 if ( $fio->{rxMatchNext} == $fio->{rxBlkCnt}
940 && !defined($blk) && !defined($newData) ) {
941 #$fio->log("$fio->{rxFile}{name}: file is unchanged");
942 # if ( $fio->{logLevel} >= 8 );
947 # need to open an output file where we will build the
950 $fio->{rxFile}{name} =~ /(.*)/s;
951 my $rxOutFileRel = "$fio->{shareM}/" . $fio->{bpc}->fileNameMangle($1);
952 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
953 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
954 $rxOutFile, $fio->{rxFile}{size},
955 $fio->{xfer}{compress});
956 $fio->log("$fio->{rxFile}{name}: opening output file $rxOutFile")
957 if ( $fio->{logLevel} >= 9 );
958 $fio->{rxOutFile} = $rxOutFile;
959 $fio->{rxOutFileRel} = $rxOutFileRel;
960 $fio->{rxDigest} = File::RsyncP::Digest->new();
961 $fio->{rxDigest}->protocol($fio->{protocol_version});
962 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
964 if ( defined($fio->{rxMatchBlk})
965 && $fio->{rxMatchBlk} != $fio->{rxMatchNext} ) {
967 # Need to copy the sequence of blocks that matched. If the file
968 # is compressed we need to make a copy of the uncompressed file,
969 # since the compressed file is not seekable. Future optimizations
970 # could include only creating an uncompressed copy if the matching
971 # blocks were not monotonic, and to only do this if there are
972 # matching blocks (eg, maybe the entire file is new).
974 my $attr = $fio->{rxLocalAttr};
976 if ( !defined($fio->{rxInFd}) && !defined($fio->{rxInData}) ) {
977 my $inPath = $attr->{fullPath};
978 $inPath = $fio->{rxHLinkAttr}{fullPath}
979 if ( defined($fio->{rxHLinkAttr}) );
980 if ( $attr->{compress} ) {
981 if ( !defined($fh = BackupPC::FileZIO->open(
984 $attr->{compress})) ) {
985 $fio->log("Can't open $inPath");
986 $fio->{stats}{errorCnt}++;
989 if ( $attr->{size} < 16 * 1024 * 1024 ) {
991 # Cache the entire old file if it is less than 16MB
994 $fio->{rxInData} = "";
995 while ( $fh->read(\$data, 16 * 1024 * 1024) > 0 ) {
996 $fio->{rxInData} .= $data;
998 $fio->log("$attr->{fullPath}: cached all $attr->{size}"
1000 if ( $fio->{logLevel} >= 9 );
1003 # Create and write a temporary output file
1005 unlink("$fio->{outDirSh}RStmp")
1006 if ( -f "$fio->{outDirSh}RStmp" );
1007 if ( open(F, "+>", "$fio->{outDirSh}RStmp") ) {
1011 while ( $fh->read(\$data, 1024 * 1024) > 0 ) {
1012 if ( syswrite(F, $data) != length($data) ) {
1013 $fio->log(sprintf("Can't write len=%d to %s",
1014 length($data) , "$fio->{outDirSh}RStmp"));
1016 $fio->{stats}{errorCnt}++;
1019 $byteCnt += length($data);
1021 $fio->{rxInFd} = *F;
1022 $fio->{rxInName} = "$fio->{outDirSh}RStmp";
1023 sysseek($fio->{rxInFd}, 0, 0);
1024 $fio->log("$attr->{fullPath}: copied $byteCnt,"
1025 . "$attr->{size} bytes to $fio->{rxInName}")
1026 if ( $fio->{logLevel} >= 9 );
1028 $fio->log("Unable to open $fio->{outDirSh}RStmp");
1030 $fio->{stats}{errorCnt}++;
1036 if ( open(F, "<", $inPath) ) {
1038 $fio->{rxInFd} = *F;
1039 $fio->{rxInName} = $attr->{fullPath};
1041 $fio->log("Unable to open $inPath");
1042 $fio->{stats}{errorCnt}++;
1047 my $lastBlk = $fio->{rxMatchNext} - 1;
1048 $fio->log("$fio->{rxFile}{name}: writing blocks $fio->{rxMatchBlk}.."
1050 if ( $fio->{logLevel} >= 9 );
1051 my $seekPosn = $fio->{rxMatchBlk} * $fio->{rxBlkSize};
1052 if ( defined($fio->{rxInFd})
1053 && !sysseek($fio->{rxInFd}, $seekPosn, 0) ) {
1054 $fio->log("Unable to seek $fio->{rxInName} to $seekPosn");
1055 $fio->{stats}{errorCnt}++;
1058 my $cnt = $fio->{rxMatchNext} - $fio->{rxMatchBlk};
1059 my($thisCnt, $len, $data);
1060 for ( my $i = 0 ; $i < $cnt ; $i += $thisCnt ) {
1061 $thisCnt = $cnt - $i;
1062 $thisCnt = 512 if ( $thisCnt > 512 );
1063 if ( $fio->{rxMatchBlk} + $i + $thisCnt == $fio->{rxBlkCnt} ) {
1064 $len = ($thisCnt - 1) * $fio->{rxBlkSize} + $fio->{rxRemainder};
1066 $len = $thisCnt * $fio->{rxBlkSize};
1068 if ( defined($fio->{rxInData}) ) {
1069 $data = substr($fio->{rxInData}, $seekPosn, $len);
1072 my $got = sysread($fio->{rxInFd}, $data, $len);
1073 if ( $got != $len ) {
1074 my $inFileSize = -s $fio->{rxInName};
1075 $fio->log("Unable to read $len bytes from $fio->{rxInName}"
1076 . " got=$got, seekPosn=$seekPosn"
1077 . " ($i,$thisCnt,$fio->{rxBlkCnt},$inFileSize"
1078 . ",$attr->{size})");
1079 $fio->{stats}{errorCnt}++;
1084 $fio->{rxOutFd}->write(\$data);
1085 $fio->{rxDigest}->add($data);
1086 $fio->{rxSize} += length($data);
1088 $fio->{rxMatchBlk} = undef;
1090 if ( defined($blk) ) {
1092 # Remember the new block number
1094 $fio->{rxMatchBlk} = $blk;
1095 $fio->{rxMatchNext} = $blk + 1;
1097 if ( defined($newData) ) {
1099 # Write the new chunk
1101 my $len = length($newData);
1102 $fio->log("$fio->{rxFile}{name}: writing $len bytes new data")
1103 if ( $fio->{logLevel} >= 9 );
1104 $fio->{rxOutFd}->write(\$newData);
1105 $fio->{rxDigest}->add($newData);
1106 $fio->{rxSize} += length($newData);
1111 # Finish up the current receive file. Returns undef if ok, -1 if not.
1112 # Returns 1 if the md4 digest doesn't match.
1116 my($fio, $md4, $phase) = @_;
1117 my $name = $1 if ( $fio->{rxFile}{name} =~ /(.*)/s );
1120 close($fio->{rxInFd}) if ( defined($fio->{rxInFd}) );
1121 unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" );
1122 $fio->{phase} = $phase;
1125 # Check the final md4 digest
1127 if ( defined($md4) ) {
1129 if ( !defined($fio->{rxDigest}) ) {
1131 # File was exact match, but we still need to verify the
1132 # MD4 checksum. Compute the md4 digest (or fetch the
1135 if ( defined(my $attr = $fio->{rxLocalAttr}) ) {
1137 # block size doesn't matter: we're only going to
1138 # fetch the md4 file digest, not the block digests.
1140 my($err, $csum, $blkSize)
1141 = BackupPC::Xfer::RsyncDigest->digestStart(
1142 $attr->{fullPath}, $attr->{size},
1143 0, 2048, $fio->{checksumSeed}, 1,
1144 $attr->{compress}, 1,
1145 $fio->{protocol_version});
1147 $fio->log("Can't open $attr->{fullPath} for MD4"
1148 . " check (err=$err, $name)");
1149 $fio->{stats}{errorCnt}++;
1151 if ( $fio->{logLevel} >= 5 ) {
1152 my($isCached, $invalid) = $csum->isCached;
1153 $fio->log("MD4 $attr->{fullPath} cache = $isCached,"
1154 . " invalid = $invalid");
1156 $newDigest = $csum->digestEnd;
1158 $fio->{rxSize} = $attr->{size};
1161 # Empty file; just create an empty file digest
1163 $fio->{rxDigest} = File::RsyncP::Digest->new();
1164 $fio->{rxDigest}->protocol($fio->{protocol_version});
1165 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
1166 $newDigest = $fio->{rxDigest}->digest;
1168 $fio->log("$name got exact match") if ( $fio->{logLevel} >= 5 );
1170 $newDigest = $fio->{rxDigest}->digest;
1172 if ( $fio->{logLevel} >= 3 ) {
1173 my $md4Str = unpack("H*", $md4);
1174 my $newStr = unpack("H*", $newDigest);
1175 $fio->log("$name got digests $md4Str vs $newStr")
1177 if ( $md4 ne $newDigest ) {
1179 $fio->log("$name: fatal error: md4 doesn't match on retry;"
1181 $fio->{stats}{errorCnt}++;
1183 $fio->log("$name: md4 doesn't match: will retry in phase 1;"
1186 if ( defined($fio->{rxOutFd}) ) {
1187 $fio->{rxOutFd}->close;
1188 unlink($fio->{rxOutFile});
1190 delete($fio->{rxFile});
1191 delete($fio->{rxOutFile});
1197 # One special case is an empty file: if the file size is
1198 # zero we need to open the output file to create it.
1200 if ( $fio->{rxSize} == 0 ) {
1201 my $rxOutFileRel = "$fio->{shareM}/"
1202 . $fio->{bpc}->fileNameMangle($name);
1203 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
1204 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
1205 $rxOutFile, $fio->{rxSize},
1206 $fio->{xfer}{compress});
1208 if ( !defined($fio->{rxOutFd}) ) {
1210 # No output file, meaning original was an exact match.
1212 $fio->log("$name: nothing to do")
1213 if ( $fio->{logLevel} >= 5 );
1214 my $attr = $fio->{rxLocalAttr};
1215 my $f = $fio->{rxFile};
1216 $fio->logFileAction("same", $f) if ( $fio->{logLevel} >= 1 );
1218 || $attr->{type} != $f->{type}
1219 || $attr->{mtime} != $f->{mtime}
1220 || $attr->{size} != $f->{size}
1221 || $attr->{uid} != $f->{uid}
1222 || $attr->{gid} != $f->{gid}
1223 || $attr->{mode} != $f->{mode}
1224 || $attr->{hlink_self} != $f->{hlink_self} ) {
1226 # In the full case, or if the attributes are different,
1227 # we need to make a link from the previous file and
1228 # set the attributes.
1230 my $rxOutFile = $fio->{outDirSh}
1231 . $fio->{bpc}->fileNameMangle($name);
1232 my($exists, $digest, $origSize, $outSize, $errs)
1233 = BackupPC::PoolWrite::LinkOrCopy(
1238 $fio->{xfer}{compress});
1240 # Cumulate the stats
1242 $fio->{stats}{TotalFileCnt}++;
1243 $fio->{stats}{TotalFileSize} += $fio->{rxSize};
1244 $fio->{stats}{ExistFileCnt}++;
1245 $fio->{stats}{ExistFileSize} += $fio->{rxSize};
1246 $fio->{stats}{ExistFileCompSize} += -s $rxOutFile;
1247 $fio->{rxFile}{size} = $fio->{rxSize};
1248 $ret = $fio->attribSet($fio->{rxFile});
1249 $fio->log(@$errs) if ( defined($errs) && @$errs );
1251 if ( !$exists && $outSize > 0 ) {
1253 # the hard link failed, most likely because the target
1254 # file has too many links. We have copied the file
1255 # instead, so add this to the new file list.
1257 my $rxOutFileRel = "$fio->{shareM}/"
1258 . $fio->{bpc}->fileNameMangle($name);
1259 $rxOutFileRel =~ s{^/+}{};
1260 my $fh = $fio->{newFilesFH};
1261 print($fh "$digest $origSize $rxOutFileRel\n")
1262 if ( defined($fh) );
1266 my $exist = $fio->processClose($fio->{rxOutFd},
1267 $fio->{rxOutFileRel},
1269 $fio->logFileAction($exist ? "pool" : "create", $fio->{rxFile})
1270 if ( $fio->{logLevel} >= 1 );
1271 $fio->{rxFile}{size} = $fio->{rxSize};
1272 $ret = $fio->attribSet($fio->{rxFile});
1274 delete($fio->{rxDigest});
1275 delete($fio->{rxInData});
1276 delete($fio->{rxFile});
1277 delete($fio->{rxOutFile});
1282 # Callback function for BackupPC::View->find. Note the order of the
1283 # first two arguments.
1287 my($a, $fio, $fList, $outputFunc) = @_;
1288 my $name = $a->{relPath};
1290 my $type = $a->{type};
1291 my $extraAttribs = {};
1293 if ( $a->{mode} & S_HLINK_TARGET ) {
1294 $a->{hlink_self} = 1;
1295 $a->{mode} &= ~S_HLINK_TARGET;
1297 $n =~ s/^\Q$fio->{xfer}{pathHdrSrc}//;
1298 $fio->log("Sending $name (remote=$n) type = $type") if ( $fio->{logLevel} >= 1 );
1299 if ( $type == BPC_FTYPE_CHARDEV
1300 || $type == BPC_FTYPE_BLOCKDEV
1301 || $type == BPC_FTYPE_SYMLINK ) {
1302 my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0, $a->{compress});
1304 if ( defined($fh) ) {
1305 $rdSize = $fh->read(\$str, $a->{size} + 1024);
1306 if ( $type == BPC_FTYPE_SYMLINK ) {
1308 # Reconstruct symbolic link
1310 $extraAttribs = { link => $str };
1311 if ( $rdSize != $a->{size} ) {
1313 $fio->log("$name: can't read exactly $a->{size} bytes");
1314 $fio->{stats}{errorCnt}++;
1316 } elsif ( $str =~ /(\d*),(\d*)/ ) {
1318 # Reconstruct char or block special major/minor device num
1320 # Note: char/block devices have $a->{size} = 0, so we
1321 # can't do an error check on $rdSize.
1324 rdev => $1 * 256 + $2,
1329 $fio->log("$name: unexpected special file contents $str");
1330 $fio->{stats}{errorCnt}++;
1335 $fio->log("$name: can't open");
1336 $fio->{stats}{errorCnt}++;
1338 } elsif ( $fio->{preserve_hard_links}
1339 && ($type == BPC_FTYPE_HARDLINK || $type == BPC_FTYPE_FILE)
1340 && ($type == BPC_FTYPE_HARDLINK
1341 || $fio->{protocol_version} < 27
1342 || $a->{hlink_self}) ) {
1344 # Fill in fake inode information so that the remote rsync
1345 # can correctly create hardlinks.
1347 $name =~ s/^\.?\/+//;
1348 my($target, $inode);
1350 if ( $type == BPC_FTYPE_HARDLINK ) {
1351 my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0,
1353 if ( defined($fh) ) {
1354 $fh->read(\$target, 65536);
1356 $target =~ s/^\.?\/+//;
1357 if ( defined($fio->{hlinkFile2Num}{$target}) ) {
1358 $inode = $fio->{hlinkFile2Num}{$target};
1360 $inode = $fio->{fileListCnt};
1361 $fio->{hlinkFile2Num}{$target} = $inode;
1364 $fio->log("$a->{fullPath}: can't open for hardlink");
1365 $fio->{stats}{errorCnt}++;
1367 } elsif ( $a->{hlink_self} ) {
1368 if ( defined($fio->{hlinkFile2Num}{$name}) ) {
1369 $inode = $fio->{hlinkFile2Num}{$name};
1371 $inode = $fio->{fileListCnt};
1372 $fio->{hlinkFile2Num}{$name} = $inode;
1375 $inode = $fio->{fileListCnt} if ( !defined($inode) );
1376 $fio->log("$name: setting inode to $inode");
1385 mode => $a->{mode} & ~S_HLINK_TARGET,
1388 mtime => $a->{mtime},
1392 my $logName = $f->{name};
1393 from_to($f->{name}, "utf8", $fio->{clientCharset})
1394 if ( $fio->{clientCharset} ne "" );
1397 $logName = "$fio->{xfer}{pathHdrDest}/$logName";
1398 $logName =~ s{//+}{/}g;
1399 $f->{name} = $logName;
1400 $fio->logFileAction("restore", $f) if ( $fio->{logLevel} >= 1 );
1402 &$outputFunc($fList->encodeData);
1406 $fio->{fileListCnt}++;
1407 if ( $type != BPC_FTYPE_DIR ) {
1408 $fio->{stats}{TotalFileCnt}++;
1409 $fio->{stats}{TotalFileSize} += $a->{size};
1415 my($fio, $flist, $outputFunc) = @_;
1418 # Populate the file list with the files requested by the user.
1419 # Since some might be directories so we call BackupPC::View::find.
1421 $fio->log("fileListSend: sending file list: "
1422 . join(" ", @{$fio->{fileList}})) if ( $fio->{logLevel} >= 4 );
1423 $fio->{fileListCnt} = 0;
1424 $fio->{hlinkFile2Num} = {};
1425 foreach my $name ( @{$fio->{fileList}} ) {
1426 $fio->{view}->find($fio->{xfer}{bkupSrcNum},
1427 $fio->{xfer}{bkupSrcShare},
1429 \&fileListEltSend, $fio, $flist, $outputFunc);
1435 my($fio, $isChild) = @_;
1438 # If we are aborting early, remove the last file since
1439 # it was not complete
1441 if ( $isChild && defined($fio->{rxFile}) ) {
1442 unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" );
1443 if ( defined($fio->{rxFile}) ) {
1444 unlink($fio->{rxOutFile});
1445 $fio->log("finish: removing in-process file $fio->{rxFile}{name}");
1450 # Flush the attributes if this is the child
1452 $fio->attribWrite(undef) if ( $isChild );
1458 # join('',@_), kill 0;