1 #============================================================= -*-perl-*-
8 # Craig Barratt <cbarratt@users.sourceforge.net>
11 # Copyright (C) 2002-2003 Craig Barratt
13 #========================================================================
15 # Version 2.1.0, released 20 Jun 2004.
17 # See http://backuppc.sourceforge.net.
19 #========================================================================
21 package BackupPC::Xfer::RsyncFileIO;
25 use BackupPC::Attrib qw(:all);
27 use BackupPC::Xfer::RsyncDigest qw(:all);
28 use BackupPC::PoolWrite;
30 use constant S_HLINK_TARGET => 0400000; # this file is hardlink target
31 use constant S_IFMT => 0170000; # type of file
32 use constant S_IFDIR => 0040000; # directory
33 use constant S_IFCHR => 0020000; # character special
34 use constant S_IFBLK => 0060000; # block special
35 use constant S_IFREG => 0100000; # regular
36 use constant S_IFLNK => 0120000; # symbolic link
37 use constant S_IFSOCK => 0140000; # socket
38 use constant S_IFIFO => 0010000; # fifo
40 use vars qw( $RsyncLibOK );
43 eval "use File::RsyncP::Digest";
46 # Rsync module doesn't exist.
56 my($class, $options) = @_;
58 return if ( !$RsyncLibOK );
63 digest => File::RsyncP::Digest->new($options->{protocol_version}),
66 logHandler => \&logHandler,
73 ExistFileCompSize => 0,
78 $fio->{shareM} = $fio->{bpc}->fileNameEltMangle($fio->{share});
79 $fio->{outDir} = "$fio->{xfer}{outDir}/new/";
80 $fio->{outDirSh} = "$fio->{outDir}/$fio->{shareM}/";
81 $fio->{view} = BackupPC::View->new($fio->{bpc}, $fio->{client},
83 $fio->{full} = $fio->{xfer}{type} eq "full" ? 1 : 0;
84 $fio->{newFilesFH} = $fio->{xfer}{newFilesFH};
85 $fio->{partialNum} = undef if ( !$fio->{full} );
90 # We publish our version to File::RsyncP. This is so File::RsyncP
91 # can provide backward compatibility to older FileIO code.
95 # undef or 1: protocol version 26, no hardlinks
96 # 2: protocol version 28, supports hardlinks
105 my($fio, $value) = @_;
107 $fio->{blockSize} = $value if ( defined($value) );
108 return $fio->{blockSize};
113 my($fio, $value) = @_;
115 if ( defined($value) ) {
116 $fio->{protocol_version} = $value;
117 $fio->{digest}->protocol($fio->{protocol_version});
119 return $fio->{protocol_version};
122 sub preserve_hard_links
124 my($fio, $value) = @_;
126 $fio->{preserve_hard_links} = $value if ( defined($value) );
127 return $fio->{preserve_hard_links};
133 $fio->{logHandler} = $sub;
134 BackupPC::Xfer::RsyncDigest->logHandlerSet($sub);
138 # Setup rsync checksum computation for the given file.
142 my($fio, $f, $needMD4, $defBlkSize, $phase) = @_;
144 $defBlkSize ||= $fio->{blockSize};
145 my $attr = $fio->attribGet($f, 1);
147 $fio->csumEnd if ( defined($fio->{csum}) );
148 return -1 if ( $attr->{type} != BPC_FTYPE_FILE );
151 # Rsync uses short checksums on the first phase. If the whole-file
152 # checksum fails, then the file is repeated with full checksums.
153 # So on phase 2 we verify the checksums if they are cached.
155 if ( ($phase > 0 || rand(1) < $fio->{cacheCheckProb})
157 && $fio->{checksumSeed} == RSYNC_CSUMSEED_CACHE ) {
158 my($err, $d, $blkSize) = BackupPC::Xfer::RsyncDigest->digestStart(
159 $attr->{fullPath}, $attr->{size}, 0,
160 $defBlkSize, $fio->{checksumSeed},
161 0, $attr->{compress}, 0);
162 my($isCached, $isInvalid) = $d->isCached;
163 if ( $fio->{logLevel} >= 5 ) {
164 $fio->log("$attr->{fullPath} verify; cached = $isCached,"
165 . " invalid = $isInvalid, phase = $phase");
167 if ( $isCached || $isInvalid ) {
168 my $ret = BackupPC::Xfer::RsyncDigest->digestAdd(
169 $attr->{fullPath}, $blkSize,
170 $fio->{checksumSeed}, 1 # verify
173 $fio->log("Bad cached digest for $attr->{fullPath} ($ret);"
175 $fio->{stats}{errorCnt}++;
177 $fio->log("$f->{name}: verified cached digest")
178 if ( $fio->{logLevel} >= 2 );
183 (my $err, $fio->{csum}, my $blkSize)
184 = BackupPC::Xfer::RsyncDigest->digestStart($attr->{fullPath},
185 $attr->{size}, 0, $defBlkSize, $fio->{checksumSeed},
186 $needMD4, $attr->{compress}, 1);
187 if ( $fio->{logLevel} >= 5 ) {
188 my($isCached, $invalid) = $fio->{csum}->isCached;
189 $fio->log("$attr->{fullPath} cache = $isCached,"
190 . " invalid = $invalid, phase = $phase");
193 $fio->log("Can't get rsync digests from $attr->{fullPath}"
194 . " (err=$err, name=$f->{name})");
195 $fio->{stats}{errorCnt}++;
203 my($fio, $num, $csumLen, $blockSize) = @_;
208 return if ( !defined($fio->{csum}) );
209 return $fio->{csum}->digestGet($num, $csumLen);
216 return if ( !defined($fio->{csum}) );
217 return $fio->{csum}->digestEnd();
224 my $attr = $fio->attribGet($f, 1);
226 $fio->readEnd if ( defined($fio->{fh}) );
227 if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath},
229 $attr->{compress})) ) {
230 $fio->log("Can't open $attr->{fullPath} (name=$f->{name})");
231 $fio->{stats}{errorCnt}++;
234 $fio->log("$f->{name}: opened for read") if ( $fio->{logLevel} >= 4 );
243 return if ( !defined($fio->{fh}) );
244 if ( $fio->{fh}->read(\$fileData, $num) <= 0 ) {
245 return $fio->readEnd;
247 $fio->log(sprintf("read returns %d bytes", length($fileData)))
248 if ( $fio->{logLevel} >= 8 );
256 return if ( !defined($fio->{fh}) );
258 $fio->log("closing $fio->{file}{name})") if ( $fio->{logLevel} >= 8 );
265 my($fio, $checksumSeed) = @_;
267 $fio->{checksumSeed} = $checksumSeed;
268 $fio->log("Checksum caching enabled (checksumSeed = $checksumSeed)")
269 if ( $fio->{logLevel} >= 1 && $checksumSeed == RSYNC_CSUMSEED_CACHE );
270 $fio->log("Checksum seed is $checksumSeed")
271 if ( $fio->{logLevel} >= 2 && $checksumSeed != RSYNC_CSUMSEED_CACHE );
276 my($fio, $localDir, $remoteDir) = @_;
278 $fio->{localDir} = $localDir;
279 $fio->{remoteDir} = $remoteDir;
284 my($fio, $share, $dir) = @_;
287 #$fio->log("viewCacheDir($share, $dir)");
288 if ( !defined($share) ) {
289 $share = $fio->{share};
290 $shareM = $fio->{shareM};
292 $shareM = $fio->{bpc}->fileNameEltMangle($share);
294 $shareM = "$shareM/$dir" if ( $dir ne "" );
295 return if ( defined($fio->{viewCache}{$shareM}) );
297 # purge old cache entries (ie: those that don't match the
298 # first part of $dir).
300 foreach my $d ( keys(%{$fio->{viewCache}}) ) {
301 delete($fio->{viewCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
304 # fetch new directory attributes
306 $fio->{viewCache}{$shareM}
307 = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir);
309 # also cache partial backup attrib data too
311 if ( defined($fio->{partialNum}) ) {
312 foreach my $d ( keys(%{$fio->{partialCache}}) ) {
313 delete($fio->{partialCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
315 $fio->{partialCache}{$shareM}
316 = $fio->{view}->dirAttrib($fio->{partialNum}, $share, $dir);
323 my($dir, $fname, $share, $shareM);
326 $fname = "$fio->{xfer}{pathHdrSrc}/$fname"
327 if ( defined($fio->{xfer}{pathHdrSrc}) );
328 $fname =~ s{//+}{/}g;
329 if ( $fname =~ m{(.*)/(.*)} ) {
330 $shareM = $fio->{shareM};
333 } elsif ( $fname ne "." ) {
334 $shareM = $fio->{shareM};
340 $fname = $fio->{share};
342 $fio->viewCacheDir($share, $dir);
343 $shareM .= "/$dir" if ( $dir ne "" );
344 if ( defined(my $attr = $fio->{viewCache}{$shareM}{$fname}) ) {
346 } elsif ( defined(my $attr = $fio->{partialCache}{$shareM}{$fname}) ) {
355 my($fio, $f, $doHardLink) = @_;
357 my($attr) = $fio->attribGetWhere($f);
358 if ( $doHardLink && $attr->{type} == BPC_FTYPE_HARDLINK ) {
359 $fio->log("$attr->{fullPath}: opening for hardlink read"
360 . " (name = $f->{name})") if ( $fio->{logLevel} >= 4 );
361 my $fh = BackupPC::FileZIO->open($attr->{fullPath}, 0,
364 if ( defined($fh) ) {
365 $fh->read(\$target, 65536);
367 $target =~ s/^\.?\/+//;
369 $fio->log("$attr->{fullPath}: can't open for hardlink read");
370 $fio->{stats}{errorCnt}++;
371 $attr->{type} = BPC_FTYPE_FILE;
374 $target = "/$target" if ( $target !~ /^\// );
375 $fio->log("$attr->{fullPath}: redirecting to $target (will trim "
376 . "$fio->{xfer}{pathHdrSrc})") if ( $fio->{logLevel} >= 4 );
377 $target =~ s/^\Q$fio->{xfer}{pathHdrSrc}//;
378 $f->{name} = $target;
379 $attr = $fio->attribGet($f);
380 $fio->log(" ... now got $attr->{fullPath}")
381 if ( $fio->{logLevel} >= 4 );
389 my $mode = $f->{mode};
391 if ( ($mode & S_IFMT) == S_IFREG ) {
392 if ( defined($f->{hlink}) && !$f->{hlink_self} ) {
393 return BPC_FTYPE_HARDLINK;
395 return BPC_FTYPE_FILE;
397 } elsif ( ($mode & S_IFMT) == S_IFDIR ) {
398 return BPC_FTYPE_DIR;
399 } elsif ( ($mode & S_IFMT) == S_IFLNK ) {
400 return BPC_FTYPE_SYMLINK;
401 } elsif ( ($mode & S_IFMT) == S_IFCHR ) {
402 return BPC_FTYPE_CHARDEV;
403 } elsif ( ($mode & S_IFMT) == S_IFBLK ) {
404 return BPC_FTYPE_BLOCKDEV;
405 } elsif ( ($mode & S_IFMT) == S_IFIFO ) {
406 return BPC_FTYPE_FIFO;
407 } elsif ( ($mode & S_IFMT) == S_IFSOCK ) {
408 return BPC_FTYPE_SOCKET;
410 return BPC_FTYPE_UNKNOWN;
415 # Set the attributes for a file. Returns non-zero on error.
419 my($fio, $f, $placeHolder) = @_;
422 if ( $f->{name} =~ m{(.*)/(.*)} ) {
424 $dir = "$fio->{shareM}/" . $1;
425 } elsif ( $f->{name} eq "." ) {
427 $file = $fio->{share};
429 $dir = $fio->{shareM};
433 if ( !defined($fio->{attribLastDir}) || $fio->{attribLastDir} ne $dir ) {
435 # Flush any directories that don't match the first part
436 # of the new directory
438 foreach my $d ( keys(%{$fio->{attrib}}) ) {
439 next if ( $d eq "" || "$dir/" =~ m{^\Q$d/} );
440 $fio->attribWrite($d);
442 $fio->{attribLastDir} = $dir;
444 if ( !exists($fio->{attrib}{$dir}) ) {
445 $fio->{attrib}{$dir} = BackupPC::Attrib->new({
446 compress => $fio->{xfer}{compress},
448 my $path = $fio->{outDir} . $dir;
449 if ( -f $fio->{attrib}{$dir}->fileName($path)
450 && !$fio->{attrib}{$dir}->read($path) ) {
451 $fio->log(sprintf("Unable to read attribute file %s",
452 $fio->{attrib}{$dir}->fileName($path)));
455 $fio->log("attribSet(dir=$dir, file=$file)") if ( $fio->{logLevel} >= 4 );
457 my $mode = $f->{mode};
459 $mode |= S_HLINK_TARGET if ( $f->{hlink_self} );
460 $fio->{attrib}{$dir}->set($file, {
461 type => $fio->mode2type($f),
465 size => $placeHolder ? -1 : $f->{size},
466 mtime => $f->{mtime},
476 if ( !defined($d) ) {
478 # flush all entries (in reverse order)
480 foreach $d ( sort({$b cmp $a} keys(%{$fio->{attrib}})) ) {
481 $fio->attribWrite($d);
485 return if ( !defined($fio->{attrib}{$d}) );
487 # Set deleted files in the attributes. Any file in the view
488 # that doesn't have attributes is flagged as deleted for
489 # incremental dumps. All files sent by rsync have attributes
490 # temporarily set so we can do deletion detection. We also
491 # prune these temporary attributes.
497 $dir = $1 if ( $d =~ m{.+?/(.*)} );
498 $fio->viewCacheDir(undef, $dir);
499 ##print("attribWrite $d,$dir\n");
500 ##$Data::Dumper::Indent = 1;
501 ##$fio->log("attribWrite $d,$dir");
502 ##$fio->log("viewCacheLogKeys = ", keys(%{$fio->{viewCache}}));
503 ##$fio->log("attribKeys = ", keys(%{$fio->{attrib}}));
504 ##print "viewCache = ", Dumper($fio->{attrib});
505 ##print "attrib = ", Dumper($fio->{attrib});
506 if ( defined($fio->{viewCache}{$d}) ) {
507 foreach my $f ( keys(%{$fio->{viewCache}{$d}}) ) {
509 $name = "$1/$name" if ( $d =~ m{.*?/(.*)} );
510 if ( defined(my $a = $fio->{attrib}{$d}->get($f)) ) {
512 # delete temporary attributes (skipped files)
514 if ( $a->{size} < 0 ) {
515 $fio->{attrib}{$d}->set($f, undef);
516 $fio->logFileAction("skip", {
517 %{$fio->{viewCache}{$d}{$f}},
519 }) if ( $fio->{logLevel} >= 2 );
521 } elsif ( !$fio->{full} ) {
522 ##print("Delete file $f\n");
523 $fio->logFileAction("delete", {
524 %{$fio->{viewCache}{$d}{$f}},
526 }) if ( $fio->{logLevel} >= 1 );
527 $fio->{attrib}{$d}->set($f, {
528 type => BPC_FTYPE_DELETED,
539 if ( $fio->{attrib}{$d}->fileCount ) {
540 my $data = $fio->{attrib}{$d}->writeData;
543 $dirM = $1 . "/" . $fio->{bpc}->fileNameMangle($2)
544 if ( $dirM =~ m{(.*?)/(.*)} );
545 my $fileName = $fio->{attrib}{$d}->fileName("$fio->{outDir}$dirM");
546 $fio->log("attribWrite(dir=$d) -> $fileName")
547 if ( $fio->{logLevel} >= 4 );
548 my $poolWrite = BackupPC::PoolWrite->new($fio->{bpc}, $fileName,
549 length($data), $fio->{xfer}{compress});
550 $poolWrite->write(\$data);
551 $fio->processClose($poolWrite, $fio->{attrib}{$d}->fileName($dirM),
554 delete($fio->{attrib}{$d});
559 my($fio, $poolWrite, $fileName, $origSize, $doStats) = @_;
560 my($exists, $digest, $outSize, $errs) = $poolWrite->close;
562 $fileName =~ s{^/+}{};
563 $fio->log(@$errs) if ( defined($errs) && @$errs );
565 $fio->{stats}{TotalFileCnt}++;
566 $fio->{stats}{TotalFileSize} += $origSize;
570 $fio->{stats}{ExistFileCnt}++;
571 $fio->{stats}{ExistFileSize} += $origSize;
572 $fio->{stats}{ExistFileCompSize} += $outSize;
574 } elsif ( $outSize > 0 ) {
575 my $fh = $fio->{newFilesFH};
576 print($fh "$digest $origSize $fileName\n") if ( defined($fh) );
578 return $exists && $origSize > 0;
585 return $fio->{stats};
589 # Make a given directory. Returns non-zero on error.
594 my $name = $1 if ( $f->{name} =~ /(.*)/ );
597 if ( $name eq "." ) {
598 $path = $fio->{outDirSh};
600 $path = $fio->{outDirSh} . $fio->{bpc}->fileNameMangle($name);
602 $fio->logFileAction("create", $f) if ( $fio->{logLevel} >= 1 );
603 $fio->log("makePath($path, 0777)") if ( $fio->{logLevel} >= 5 );
604 $path = $1 if ( $path =~ /(.*)/ );
605 File::Path::mkpath($path, 0, 0777) if ( !-d $path );
606 return $fio->attribSet($f) if ( -d $path );
607 $fio->log("Can't create directory $path");
608 $fio->{stats}{errorCnt}++;
613 # Make a special file. Returns non-zero on error.
618 my $name = $1 if ( $f->{name} =~ /(.*)/ );
619 my $fNameM = $fio->{bpc}->fileNameMangle($name);
620 my $path = $fio->{outDirSh} . $fNameM;
621 my $attr = $fio->attribGet($f);
623 my $type = $fio->mode2type($f);
625 $fio->log("makeSpecial($path, $type, $f->{mode})")
626 if ( $fio->{logLevel} >= 5 );
627 if ( $type == BPC_FTYPE_CHARDEV || $type == BPC_FTYPE_BLOCKDEV ) {
628 my($major, $minor, $fh, $fileData);
630 if ( defined($f->{rdev_major}) ) {
631 $major = $f->{rdev_major};
632 $minor = $f->{rdev_minor};
634 $major = $f->{rdev} >> 8;
635 $minor = $f->{rdev} & 0xff;
637 $str = "$major,$minor";
638 } elsif ( ($f->{mode} & S_IFMT) == S_IFLNK ) {
640 } elsif ( ($f->{mode} & S_IFMT) == S_IFREG ) {
644 if ( !defined($f->{hlink}) ) {
645 $fio->log("Error: makeSpecial($path, $type, $f->{mode}) called"
646 . " on a regular non-hardlink file");
652 # Now see if the file is different, or this is a full, in which
653 # case we create the new file.
658 || $attr->{type} != $type
659 || $attr->{mtime} != $f->{mtime}
660 || $attr->{size} != $f->{size}
661 || $attr->{uid} != $f->{uid}
662 || $attr->{gid} != $f->{gid}
663 || $attr->{mode} != $f->{mode}
664 || !defined($fh = BackupPC::FileZIO->open($attr->{fullPath}, 0,
666 || $fh->read(\$fileData, length($str) + 1) != length($str)
667 || $fileData ne $str ) {
668 $fh->close if ( defined($fh) );
669 $fh = BackupPC::PoolWrite->new($fio->{bpc}, $path,
670 length($str), $fio->{xfer}{compress});
672 my $exist = $fio->processClose($fh, "$fio->{shareM}/$fNameM",
674 $fio->logFileAction($exist ? "pool" : "create", $f)
675 if ( $fio->{logLevel} >= 1 );
676 return $fio->attribSet($f);
678 $fio->logFileAction("skip", $f) if ( $fio->{logLevel} >= 2 );
680 $fh->close if ( defined($fh) );
684 # Make a hardlink. Returns non-zero on error.
685 # This actually gets called twice for each hardlink.
686 # Once as the file list is processed, and again at
687 # the end. BackupPC does them as it goes (since it is
688 # just saving the hardlink info and not actually making
693 my($fio, $f, $end) = @_;
696 return $fio->makeSpecial($f) if ( !$f->{hlink_self} );
701 my($fio, $path) = @_;
703 $fio->log("Unexpected call BackupPC::Xfer::RsyncFileIO->unlink($path)");
707 # Default log handler
713 print(STDERR $str, "\n");
717 # Handle one or more log messages
721 my($fio, @logStr) = @_;
723 foreach my $str ( @logStr ) {
724 next if ( $str eq "" );
725 $fio->{logHandler}($str);
730 # Generate a log file message for a completed file
734 my($fio, $action, $f) = @_;
735 my $owner = "$f->{uid}/$f->{gid}";
736 my $type = (("", "p", "c", "", "d", "", "b", "", "", "", "l", "", "s"))
737 [($f->{mode} & S_IFMT) >> 12];
740 if ( ($f->{mode} & S_IFMT) == S_IFLNK ) {
741 $link = " -> $f->{link}";
742 } if ( ($f->{mode} & S_IFMT) == S_IFREG
743 && defined($f->{hlink}) && !$f->{hlink_self} ) {
744 $link = " -> $f->{hlink}";
747 $fio->log(sprintf(" %-6s %1s%4o %9s %11.0f %s%s",
758 # If there is a partial and we are doing a full, we do an incremental
759 # against the partial and a full against the rest. This subroutine
760 # is how we tell File::RsyncP which files to ignore attributes on
761 # (ie: against the partial dump we do consider the attributes, but
762 # otherwise we ignore attributes).
768 return if ( !defined($fio->{partialNum}) );
769 my($attr, $isPartial) = $fio->attribGetWhere($f);
770 $fio->log("$f->{name}: just checking attributes from partial")
771 if ( $isPartial && $fio->{logLevel} >= 5 );
776 # This is called by File::RsyncP when a file is skipped because the
781 my($fio, $f, $attr) = @_;
784 # Unless this is a partial, this is normal so ignore it.
786 return if ( !defined($fio->{partialNum}) );
788 $fio->log("$f->{name}: skipped in partial; adding link")
789 if ( $fio->{logLevel} >= 5 );
790 $fio->{rxLocalAttr} = $attr;
792 $fio->{rxSize} = $attr->{size};
793 delete($fio->{rxInFd});
794 delete($fio->{rxOutFd});
795 delete($fio->{rxDigest});
796 delete($fio->{rxInData});
797 return $fio->fileDeltaRxDone();
801 # Start receive of file deltas for a particular file.
805 my($fio, $f, $cnt, $size, $remainder) = @_;
807 $fio->{rxFile} = $f; # remote file attributes
808 $fio->{rxLocalAttr} = $fio->attribGet($f); # local file attributes
809 $fio->{rxBlkCnt} = $cnt; # how many blocks we will receive
810 $fio->{rxBlkSize} = $size; # block size
811 $fio->{rxRemainder} = $remainder; # size of the last block
812 $fio->{rxMatchBlk} = 0; # current start of match
813 $fio->{rxMatchNext} = 0; # current next block of match
814 $fio->{rxSize} = 0; # size of received file
815 my $rxSize = $cnt > 0 ? ($cnt - 1) * $size + $remainder : 0;
816 if ( $fio->{rxFile}{size} != $rxSize ) {
817 $fio->{rxMatchBlk} = undef; # size different, so no file match
818 $fio->log("$fio->{rxFile}{name}: size doesn't match"
819 . " ($fio->{rxFile}{size} vs $rxSize)")
820 if ( $fio->{logLevel} >= 5 );
823 # If compression was off and now on, or on and now off, then
824 # don't do an exact match.
826 if ( defined($fio->{rxLocalAttr})
827 && !$fio->{rxLocalAttr}{compress} != !$fio->{xfer}{compress} ) {
828 $fio->{rxMatchBlk} = undef; # compression changed, so no file match
829 $fio->log("$fio->{rxFile}{name}: compression changed, so no match"
830 . " ($fio->{rxLocalAttr}{compress} vs $fio->{xfer}{compress})")
831 if ( $fio->{logLevel} >= 4 );
833 delete($fio->{rxInFd});
834 delete($fio->{rxOutFd});
835 delete($fio->{rxDigest});
836 delete($fio->{rxInData});
840 # Process the next file delta for the current file. Returns 0 if ok,
841 # -1 if not. Must be called with either a block number, $blk, or new data,
842 # $newData, (not both) defined.
846 my($fio, $blk, $newData) = @_;
848 if ( defined($blk) ) {
849 if ( defined($fio->{rxMatchBlk}) && $fio->{rxMatchNext} == $blk ) {
851 # got the next block in order; just keep track.
853 $fio->{rxMatchNext}++;
857 my $newDataLen = length($newData);
858 $fio->log("$fio->{rxFile}{name}: blk=$blk, newData=$newDataLen, rxMatchBlk=$fio->{rxMatchBlk}, rxMatchNext=$fio->{rxMatchNext}")
859 if ( $fio->{logLevel} >= 8 );
860 if ( !defined($fio->{rxOutFd}) ) {
862 # maybe the file has no changes
864 if ( $fio->{rxMatchNext} == $fio->{rxBlkCnt}
865 && !defined($blk) && !defined($newData) ) {
866 #$fio->log("$fio->{rxFile}{name}: file is unchanged");
867 # if ( $fio->{logLevel} >= 8 );
872 # need to open an output file where we will build the
875 $fio->{rxFile}{name} =~ /(.*)/;
876 my $rxOutFileRel = "$fio->{shareM}/" . $fio->{bpc}->fileNameMangle($1);
877 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
878 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
879 $rxOutFile, $fio->{rxFile}{size},
880 $fio->{xfer}{compress});
881 $fio->log("$fio->{rxFile}{name}: opening output file $rxOutFile")
882 if ( $fio->{logLevel} >= 9 );
883 $fio->{rxOutFile} = $rxOutFile;
884 $fio->{rxOutFileRel} = $rxOutFileRel;
885 $fio->{rxDigest} = File::RsyncP::Digest->new($fio->{protocol_version});
886 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
888 if ( defined($fio->{rxMatchBlk})
889 && $fio->{rxMatchBlk} != $fio->{rxMatchNext} ) {
891 # Need to copy the sequence of blocks that matched. If the file
892 # is compressed we need to make a copy of the uncompressed file,
893 # since the compressed file is not seekable. Future optimizations
894 # could include only creating an uncompressed copy if the matching
895 # blocks were not monotonic, and to only do this if there are
896 # matching blocks (eg, maybe the entire file is new).
898 my $attr = $fio->{rxLocalAttr};
900 if ( !defined($fio->{rxInFd}) && !defined($fio->{rxInData}) ) {
901 if ( $attr->{compress} ) {
902 if ( !defined($fh = BackupPC::FileZIO->open(
905 $attr->{compress})) ) {
906 $fio->log("Can't open $attr->{fullPath}");
907 $fio->{stats}{errorCnt}++;
910 if ( $attr->{size} < 16 * 1024 * 1024 ) {
912 # Cache the entire old file if it is less than 16MB
915 $fio->{rxInData} = "";
916 while ( $fh->read(\$data, 16 * 1024 * 1024) > 0 ) {
917 $fio->{rxInData} .= $data;
919 $fio->log("$attr->{fullPath}: cached all $attr->{size}"
921 if ( $fio->{logLevel} >= 9 );
924 # Create and write a temporary output file
926 unlink("$fio->{outDirSh}RStmp")
927 if ( -f "$fio->{outDirSh}RStmp" );
928 if ( open(F, "+>", "$fio->{outDirSh}RStmp") ) {
932 while ( $fh->read(\$data, 1024 * 1024) > 0 ) {
933 if ( syswrite(F, $data) != length($data) ) {
934 $fio->log(sprintf("Can't write len=%d to %s",
935 length($data) , "$fio->{outDirSh}RStmp"));
937 $fio->{stats}{errorCnt}++;
940 $byteCnt += length($data);
943 $fio->{rxInName} = "$fio->{outDirSh}RStmp";
944 sysseek($fio->{rxInFd}, 0, 0);
945 $fio->log("$attr->{fullPath}: copied $byteCnt,"
946 . "$attr->{size} bytes to $fio->{rxInName}")
947 if ( $fio->{logLevel} >= 9 );
949 $fio->log("Unable to open $fio->{outDirSh}RStmp");
951 $fio->{stats}{errorCnt}++;
957 if ( open(F, "<", $attr->{fullPath}) ) {
960 $fio->{rxInName} = $attr->{fullPath};
962 $fio->log("Unable to open $attr->{fullPath}");
963 $fio->{stats}{errorCnt}++;
968 my $lastBlk = $fio->{rxMatchNext} - 1;
969 $fio->log("$fio->{rxFile}{name}: writing blocks $fio->{rxMatchBlk}.."
971 if ( $fio->{logLevel} >= 9 );
972 my $seekPosn = $fio->{rxMatchBlk} * $fio->{rxBlkSize};
973 if ( defined($fio->{rxInFd})
974 && !sysseek($fio->{rxInFd}, $seekPosn, 0) ) {
975 $fio->log("Unable to seek $attr->{rxInName} to $seekPosn");
976 $fio->{stats}{errorCnt}++;
979 my $cnt = $fio->{rxMatchNext} - $fio->{rxMatchBlk};
980 my($thisCnt, $len, $data);
981 for ( my $i = 0 ; $i < $cnt ; $i += $thisCnt ) {
982 $thisCnt = $cnt - $i;
983 $thisCnt = 512 if ( $thisCnt > 512 );
984 if ( $fio->{rxMatchBlk} + $i + $thisCnt == $fio->{rxBlkCnt} ) {
985 $len = ($thisCnt - 1) * $fio->{rxBlkSize} + $fio->{rxRemainder};
987 $len = $thisCnt * $fio->{rxBlkSize};
989 if ( defined($fio->{rxInData}) ) {
990 $data = substr($fio->{rxInData}, $seekPosn, $len);
993 my $got = sysread($fio->{rxInFd}, $data, $len);
994 if ( $got != $len ) {
995 my $inFileSize = -s $fio->{rxInName};
996 $fio->log("Unable to read $len bytes from $fio->{rxInName}"
997 . " got=$got, seekPosn=$seekPosn"
998 . " ($i,$thisCnt,$fio->{rxBlkCnt},$inFileSize"
999 . ",$attr->{size})");
1000 $fio->{stats}{errorCnt}++;
1005 $fio->{rxOutFd}->write(\$data);
1006 $fio->{rxDigest}->add($data);
1007 $fio->{rxSize} += length($data);
1009 $fio->{rxMatchBlk} = undef;
1011 if ( defined($blk) ) {
1013 # Remember the new block number
1015 $fio->{rxMatchBlk} = $blk;
1016 $fio->{rxMatchNext} = $blk + 1;
1018 if ( defined($newData) ) {
1020 # Write the new chunk
1022 my $len = length($newData);
1023 $fio->log("$fio->{rxFile}{name}: writing $len bytes new data")
1024 if ( $fio->{logLevel} >= 9 );
1025 $fio->{rxOutFd}->write(\$newData);
1026 $fio->{rxDigest}->add($newData);
1027 $fio->{rxSize} += length($newData);
1032 # Finish up the current receive file. Returns undef if ok, -1 if not.
1033 # Returns 1 if the md4 digest doesn't match.
1037 my($fio, $md4, $phase) = @_;
1038 my $name = $1 if ( $fio->{rxFile}{name} =~ /(.*)/ );
1041 close($fio->{rxInFd}) if ( defined($fio->{rxInFd}) );
1042 unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" );
1045 # Check the final md4 digest
1047 if ( defined($md4) ) {
1049 if ( !defined($fio->{rxDigest}) ) {
1051 # File was exact match, but we still need to verify the
1052 # MD4 checksum. Compute the md4 digest (or fetch the
1055 if ( defined(my $attr = $fio->{rxLocalAttr}) ) {
1057 # block size doesn't matter: we're only going to
1058 # fetch the md4 file digest, not the block digests.
1060 my($err, $csum, $blkSize)
1061 = BackupPC::Xfer::RsyncDigest->digestStart(
1062 $attr->{fullPath}, $attr->{size},
1063 0, 2048, $fio->{checksumSeed}, 1,
1064 $attr->{compress}, 1);
1066 $fio->log("Can't open $attr->{fullPath} for MD4"
1067 . " check (err=$err, $name)");
1068 $fio->{stats}{errorCnt}++;
1070 if ( $fio->{logLevel} >= 5 ) {
1071 my($isCached, $invalid) = $csum->isCached;
1072 $fio->log("MD4 $attr->{fullPath} cache = $isCached,"
1073 . " invalid = $invalid");
1075 $newDigest = $csum->digestEnd;
1077 $fio->{rxSize} = $attr->{size};
1080 # Empty file; just create an empty file digest
1082 $fio->{rxDigest} = File::RsyncP::Digest->new($fio->{protocol_version});
1083 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
1084 $newDigest = $fio->{rxDigest}->digest;
1086 $fio->log("$name got exact match") if ( $fio->{logLevel} >= 5 );
1088 $newDigest = $fio->{rxDigest}->digest;
1090 if ( $fio->{logLevel} >= 3 ) {
1091 my $md4Str = unpack("H*", $md4);
1092 my $newStr = unpack("H*", $newDigest);
1093 $fio->log("$name got digests $md4Str vs $newStr")
1095 if ( $md4 ne $newDigest ) {
1097 $fio->log("$name: fatal error: md4 doesn't match on retry;"
1100 $fio->log("$name: md4 doesn't match: will retry in phase 1;"
1103 $fio->{stats}{errorCnt}++;
1104 if ( defined($fio->{rxOutFd}) ) {
1105 $fio->{rxOutFd}->close;
1106 unlink($fio->{rxOutFile});
1108 delete($fio->{rxFile});
1109 delete($fio->{rxOutFile});
1115 # One special case is an empty file: if the file size is
1116 # zero we need to open the output file to create it.
1118 if ( $fio->{rxSize} == 0 ) {
1119 my $rxOutFileRel = "$fio->{shareM}/"
1120 . $fio->{bpc}->fileNameMangle($name);
1121 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
1122 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
1123 $rxOutFile, $fio->{rxSize},
1124 $fio->{xfer}{compress});
1126 if ( !defined($fio->{rxOutFd}) ) {
1128 # No output file, meaning original was an exact match.
1130 $fio->log("$name: nothing to do")
1131 if ( $fio->{logLevel} >= 5 );
1132 my $attr = $fio->{rxLocalAttr};
1133 my $f = $fio->{rxFile};
1134 $fio->logFileAction("same", $f) if ( $fio->{logLevel} >= 1 );
1136 || $attr->{type} != $f->{type}
1137 || $attr->{mtime} != $f->{mtime}
1138 || $attr->{size} != $f->{size}
1139 || $attr->{gid} != $f->{gid}
1140 || $attr->{mode} != $f->{mode} ) {
1142 # In the full case, or if the attributes are different,
1143 # we need to make a link from the previous file and
1144 # set the attributes.
1146 my $rxOutFile = $fio->{outDirSh}
1147 . $fio->{bpc}->fileNameMangle($name);
1148 if ( !link($attr->{fullPath}, $rxOutFile) ) {
1149 $fio->log("Unable to link $attr->{fullPath} to $rxOutFile");
1150 $fio->{stats}{errorCnt}++;
1154 # Cumulate the stats
1156 $fio->{stats}{TotalFileCnt}++;
1157 $fio->{stats}{TotalFileSize} += $fio->{rxSize};
1158 $fio->{stats}{ExistFileCnt}++;
1159 $fio->{stats}{ExistFileSize} += $fio->{rxSize};
1160 $fio->{stats}{ExistFileCompSize} += -s $rxOutFile;
1161 $fio->{rxFile}{size} = $fio->{rxSize};
1162 $ret = $fio->attribSet($fio->{rxFile});
1166 my $exist = $fio->processClose($fio->{rxOutFd},
1167 $fio->{rxOutFileRel},
1169 $fio->logFileAction($exist ? "pool" : "create", $fio->{rxFile})
1170 if ( $fio->{logLevel} >= 1 );
1171 $fio->{rxFile}{size} = $fio->{rxSize};
1172 $ret = $fio->attribSet($fio->{rxFile});
1174 delete($fio->{rxDigest});
1175 delete($fio->{rxInData});
1176 delete($fio->{rxFile});
1177 delete($fio->{rxOutFile});
1182 # Callback function for BackupPC::View->find. Note the order of the
1183 # first two arguments.
1187 my($a, $fio, $fList, $outputFunc) = @_;
1188 my $name = $a->{relPath};
1190 my $type = $a->{type};
1191 my $extraAttribs = {};
1193 $n =~ s/^\Q$fio->{xfer}{pathHdrSrc}//;
1194 $fio->log("Sending $name (remote=$n) type = $type") if ( $fio->{logLevel} >= 1 );
1195 if ( $type == BPC_FTYPE_CHARDEV
1196 || $type == BPC_FTYPE_BLOCKDEV
1197 || $type == BPC_FTYPE_SYMLINK ) {
1198 my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0, $a->{compress});
1200 if ( defined($fh) ) {
1201 $rdSize = $fh->read(\$str, $a->{size} + 1024);
1202 if ( $type == BPC_FTYPE_SYMLINK ) {
1204 # Reconstruct symbolic link
1206 $extraAttribs = { link => $str };
1207 if ( $rdSize != $a->{size} ) {
1209 $fio->log("$name: can't read exactly $a->{size} bytes");
1210 $fio->{stats}{errorCnt}++;
1212 } elsif ( $str =~ /(\d*),(\d*)/ ) {
1214 # Reconstruct char or block special major/minor device num
1216 # Note: char/block devices have $a->{size} = 0, so we
1217 # can't do an error check on $rdSize.
1220 rdev => $1 * 256 + $2,
1225 $fio->log("$name: unexpected special file contents $str");
1226 $fio->{stats}{errorCnt}++;
1231 $fio->log("$name: can't open");
1232 $fio->{stats}{errorCnt}++;
1234 } elsif ( $fio->{preserve_hard_links}
1235 && ($type == BPC_FTYPE_HARDLINK || $type == BPC_FTYPE_FILE)
1236 && ($type == BPC_FTYPE_HARDLINK
1237 || $fio->{protocol_version} < 27
1238 || $a->{mode} & S_HLINK_TARGET ) ) {
1240 # Fill in fake inode information so that the remote rsync
1241 # can correctly create hardlinks.
1243 $name =~ s/^\.?\/+//;
1244 my($target, $inode);
1246 if ( $type == BPC_FTYPE_HARDLINK ) {
1247 my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0,
1249 if ( defined($fh) ) {
1250 $fh->read(\$target, 65536);
1252 $target =~ s/^\.?\/+//;
1253 if ( defined($fio->{hlinkFile2Num}{$target}) ) {
1254 $inode = $fio->{hlinkFile2Num}{$target};
1256 $inode = $fio->{fileListCnt};
1257 $fio->{hlinkFile2Num}{$target} = $inode;
1260 $fio->log("$a->{fullPath}: can't open for hardlink");
1261 $fio->{stats}{errorCnt}++;
1263 } elsif ( $a->{mode} & S_HLINK_TARGET ) {
1264 if ( defined($fio->{hlinkFile2Num}{$name}) ) {
1265 $inode = $fio->{hlinkFile2Num}{$name};
1267 $inode = $fio->{fileListCnt};
1268 $fio->{hlinkFile2Num}{$name} = $inode;
1271 $inode = $fio->{fileListCnt} if ( !defined($inode) );
1272 $fio->log("$name: setting inode to $inode");
1281 mode => $a->{mode} & ~S_HLINK_TARGET,
1284 mtime => $a->{mtime},
1289 $f->{name} = "$fio->{xfer}{pathHdrDest}/$f->{name}";
1290 $f->{name} =~ s{//+}{/}g;
1291 $fio->logFileAction("restore", $f) if ( $fio->{logLevel} >= 1 );
1292 &$outputFunc($fList->encodeData);
1296 $fio->{fileListCnt}++;
1297 if ( $type != BPC_FTYPE_DIR ) {
1298 $fio->{stats}{TotalFileCnt}++;
1299 $fio->{stats}{TotalFileSize} += $a->{size};
1305 my($fio, $flist, $outputFunc) = @_;
1308 # Populate the file list with the files requested by the user.
1309 # Since some might be directories so we call BackupPC::View::find.
1311 $fio->log("fileListSend: sending file list: "
1312 . join(" ", @{$fio->{fileList}})) if ( $fio->{logLevel} >= 4 );
1313 $fio->{fileListCnt} = 0;
1314 $fio->{hlinkFile2Num} = {};
1315 foreach my $name ( @{$fio->{fileList}} ) {
1316 $fio->{view}->find($fio->{xfer}{bkupSrcNum},
1317 $fio->{xfer}{bkupSrcShare},
1319 \&fileListEltSend, $fio, $flist, $outputFunc);
1325 my($fio, $isChild) = @_;
1328 # If we are aborting early, remove the last file since
1329 # it was not complete
1331 if ( $isChild && defined($fio->{rxFile}) ) {
1332 unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" );
1333 if ( defined($fio->{rxFile}) ) {
1334 unlink($fio->{rxOutFile});
1335 $fio->log("finish: removing in-process file $fio->{rxFile}{name}");
1340 # Flush the attributes if this is the child
1342 $fio->attribWrite(undef) if ( $isChild );
1348 # join('',@_), kill 0;