1 #============================================================= -*-perl-*-
8 # Craig Barratt <cbarratt@users.sourceforge.net>
11 # Copyright (C) 2002-2003 Craig Barratt
13 #========================================================================
15 # Version 3.0.0beta2, released 11 Nov 2006.
17 # See http://backuppc.sourceforge.net.
19 #========================================================================
21 package BackupPC::Xfer::RsyncFileIO;
25 use Encode qw/from_to/;
26 use BackupPC::Attrib qw(:all);
28 use BackupPC::Xfer::RsyncDigest qw(:all);
29 use BackupPC::PoolWrite;
31 use constant S_HLINK_TARGET => 0400000; # this file is hardlink target
32 use constant S_IFMT => 0170000; # type of file
33 use constant S_IFDIR => 0040000; # directory
34 use constant S_IFCHR => 0020000; # character special
35 use constant S_IFBLK => 0060000; # block special
36 use constant S_IFREG => 0100000; # regular
37 use constant S_IFLNK => 0120000; # symbolic link
38 use constant S_IFSOCK => 0140000; # socket
39 use constant S_IFIFO => 0010000; # fifo
41 use vars qw( $RsyncLibOK );
44 eval "use File::RsyncP::Digest";
47 # Rsync module doesn't exist.
57 my($class, $options) = @_;
59 return if ( !$RsyncLibOK );
64 digest => File::RsyncP::Digest->new(),
67 logHandler => \&logHandler,
74 ExistFileCompSize => 0,
79 $fio->{digest}->protocol($fio->{protocol_version});
80 $fio->{shareM} = $fio->{bpc}->fileNameEltMangle($fio->{share});
81 $fio->{outDir} = "$fio->{xfer}{outDir}/new/";
82 $fio->{outDirSh} = "$fio->{outDir}/$fio->{shareM}/";
83 $fio->{view} = BackupPC::View->new($fio->{bpc}, $fio->{client},
85 $fio->{full} = $fio->{xfer}{type} eq "full" ? 1 : 0;
86 $fio->{newFilesFH} = $fio->{xfer}{newFilesFH};
87 $fio->{partialNum} = undef if ( !$fio->{full} );
92 # We publish our version to File::RsyncP. This is so File::RsyncP
93 # can provide backward compatibility to older FileIO code.
97 # undef or 1: protocol version 26, no hardlinks
98 # 2: protocol version 28, supports hardlinks
107 my($fio, $value) = @_;
109 $fio->{blockSize} = $value if ( defined($value) );
110 return $fio->{blockSize};
115 my($fio, $value) = @_;
117 if ( defined($value) ) {
118 $fio->{protocol_version} = $value;
119 $fio->{digest}->protocol($fio->{protocol_version});
121 return $fio->{protocol_version};
124 sub preserve_hard_links
126 my($fio, $value) = @_;
128 $fio->{preserve_hard_links} = $value if ( defined($value) );
129 return $fio->{preserve_hard_links};
135 $fio->{logHandler} = $sub;
136 BackupPC::Xfer::RsyncDigest->logHandlerSet($sub);
140 # Setup rsync checksum computation for the given file.
144 my($fio, $f, $needMD4, $defBlkSize, $phase) = @_;
146 $defBlkSize ||= $fio->{blockSize};
147 my $attr = $fio->attribGet($f, 1);
149 $fio->csumEnd if ( defined($fio->{csum}) );
150 return -1 if ( $attr->{type} != BPC_FTYPE_FILE );
153 # Rsync uses short checksums on the first phase. If the whole-file
154 # checksum fails, then the file is repeated with full checksums.
155 # So on phase 2 we verify the checksums if they are cached.
157 if ( ($phase > 0 || rand(1) < $fio->{cacheCheckProb})
159 && $fio->{checksumSeed} == RSYNC_CSUMSEED_CACHE ) {
160 my($err, $d, $blkSize) = BackupPC::Xfer::RsyncDigest->digestStart(
161 $attr->{fullPath}, $attr->{size}, 0,
162 $defBlkSize, $fio->{checksumSeed},
163 0, $attr->{compress}, 0,
164 $fio->{protocol_version});
165 my($isCached, $isInvalid) = $d->isCached;
166 if ( $fio->{logLevel} >= 5 ) {
167 $fio->log("$attr->{fullPath} verify; cached = $isCached,"
168 . " invalid = $isInvalid, phase = $phase");
170 if ( $isCached || $isInvalid ) {
171 my $ret = BackupPC::Xfer::RsyncDigest->digestAdd(
172 $attr->{fullPath}, $blkSize,
173 $fio->{checksumSeed}, 1 # verify
176 $fio->log("Bad cached digest for $attr->{fullPath} ($ret);"
178 $fio->{stats}{errorCnt}++;
180 $fio->log("$f->{name}: verified cached digest")
181 if ( $fio->{logLevel} >= 2 );
186 (my $err, $fio->{csum}, my $blkSize)
187 = BackupPC::Xfer::RsyncDigest->digestStart($attr->{fullPath},
188 $attr->{size}, 0, $defBlkSize, $fio->{checksumSeed},
189 $needMD4, $attr->{compress}, 1,
190 $fio->{protocol_version});
191 if ( $fio->{logLevel} >= 5 ) {
192 my($isCached, $invalid) = $fio->{csum}->isCached;
193 $fio->log("$attr->{fullPath} cache = $isCached,"
194 . " invalid = $invalid, phase = $phase");
197 $fio->log("Can't get rsync digests from $attr->{fullPath}"
198 . " (err=$err, name=$f->{name})");
199 $fio->{stats}{errorCnt}++;
207 my($fio, $num, $csumLen, $blockSize) = @_;
212 return if ( !defined($fio->{csum}) );
213 return $fio->{csum}->digestGet($num, $csumLen);
220 return if ( !defined($fio->{csum}) );
221 return $fio->{csum}->digestEnd();
228 my $attr = $fio->attribGet($f, 1);
230 $fio->readEnd if ( defined($fio->{fh}) );
231 if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath},
233 $attr->{compress})) ) {
234 $fio->log("Can't open $attr->{fullPath} (name=$f->{name})");
235 $fio->{stats}{errorCnt}++;
238 $fio->log("$f->{name}: opened for read") if ( $fio->{logLevel} >= 4 );
247 return if ( !defined($fio->{fh}) );
248 if ( $fio->{fh}->read(\$fileData, $num) <= 0 ) {
249 return $fio->readEnd;
251 $fio->log(sprintf("read returns %d bytes", length($fileData)))
252 if ( $fio->{logLevel} >= 8 );
260 return if ( !defined($fio->{fh}) );
262 $fio->log("closing $fio->{file}{name})") if ( $fio->{logLevel} >= 8 );
269 my($fio, $checksumSeed) = @_;
271 $fio->{checksumSeed} = $checksumSeed;
272 $fio->log("Checksum caching enabled (checksumSeed = $checksumSeed)")
273 if ( $fio->{logLevel} >= 1 && $checksumSeed == RSYNC_CSUMSEED_CACHE );
274 $fio->log("Checksum seed is $checksumSeed")
275 if ( $fio->{logLevel} >= 2 && $checksumSeed != RSYNC_CSUMSEED_CACHE );
280 my($fio, $localDir, $remoteDir) = @_;
282 $fio->{localDir} = $localDir;
283 $fio->{remoteDir} = $remoteDir;
288 my($fio, $share, $dir) = @_;
291 #$fio->log("viewCacheDir($share, $dir)");
292 if ( !defined($share) ) {
293 $share = $fio->{share};
294 $shareM = $fio->{shareM};
296 $shareM = $fio->{bpc}->fileNameEltMangle($share);
298 $shareM = "$shareM/$dir" if ( $dir ne "" );
299 return if ( defined($fio->{viewCache}{$shareM}) );
301 # purge old cache entries (ie: those that don't match the
302 # first part of $dir).
304 foreach my $d ( keys(%{$fio->{viewCache}}) ) {
305 delete($fio->{viewCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
308 # fetch new directory attributes
310 $fio->{viewCache}{$shareM}
311 = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir);
313 # also cache partial backup attrib data too
315 if ( defined($fio->{partialNum}) ) {
316 foreach my $d ( keys(%{$fio->{partialCache}}) ) {
317 delete($fio->{partialCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
319 $fio->{partialCache}{$shareM}
320 = $fio->{view}->dirAttrib($fio->{partialNum}, $share, $dir);
326 my($fio, $f, $noCache) = @_;
327 my($dir, $fname, $share, $shareM, $partial, $attr);
330 $fname = "$fio->{xfer}{pathHdrSrc}/$fname"
331 if ( defined($fio->{xfer}{pathHdrSrc}) );
332 $fname =~ s{//+}{/}g;
333 if ( $fname =~ m{(.*)/(.*)}s ) {
334 $shareM = $fio->{shareM};
337 } elsif ( $fname ne "." ) {
338 $shareM = $fio->{shareM};
344 $fname = $fio->{share};
346 $shareM .= "/$dir" if ( $dir ne "" );
349 $share = $fio->{share} if ( !defined($share) );
350 my $dirAttr = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir);
351 $attr = $dirAttr->{$fname};
353 $fio->viewCacheDir($share, $dir);
354 if ( defined($attr = $fio->{viewCache}{$shareM}{$fname}) ) {
356 } elsif ( defined($attr = $fio->{partialCache}{$shareM}{$fname}) ) {
361 if ( $attr->{mode} & S_HLINK_TARGET ) {
362 $attr->{hlink_self} = 1;
363 $attr->{mode} &= ~S_HLINK_TARGET;
366 return ($attr, $partial);
371 my($fio, $f, $doHardLink) = @_;
373 my($attr) = $fio->attribGetWhere($f);
374 if ( $doHardLink && $attr->{type} == BPC_FTYPE_HARDLINK ) {
375 $fio->log("$attr->{fullPath}: opening for hardlink read"
376 . " (name = $f->{name})") if ( $fio->{logLevel} >= 4 );
377 my $fh = BackupPC::FileZIO->open($attr->{fullPath}, 0,
380 if ( defined($fh) ) {
381 $fh->read(\$target, 65536);
383 $target =~ s/^\.?\/+//;
385 $fio->log("$attr->{fullPath}: can't open for hardlink read");
386 $fio->{stats}{errorCnt}++;
387 $attr->{type} = BPC_FTYPE_FILE;
390 $target = "/$target" if ( $target !~ /^\// );
391 $fio->log("$attr->{fullPath}: redirecting to $target (will trim "
392 . "$fio->{xfer}{pathHdrSrc})") if ( $fio->{logLevel} >= 4 );
393 $target =~ s/^\Q$fio->{xfer}{pathHdrSrc}//;
396 # Note: overwrites name to point to real file
398 $f->{name} = $target;
399 ($attr) = $fio->attribGetWhere($f, 1);
400 $fio->log(" ... now got $attr->{fullPath}")
401 if ( $fio->{logLevel} >= 4 );
409 my $mode = $f->{mode};
411 if ( ($mode & S_IFMT) == S_IFREG ) {
412 if ( defined($f->{hlink}) && !$f->{hlink_self} ) {
413 return BPC_FTYPE_HARDLINK;
415 return BPC_FTYPE_FILE;
417 } elsif ( ($mode & S_IFMT) == S_IFDIR ) {
418 return BPC_FTYPE_DIR;
419 } elsif ( ($mode & S_IFMT) == S_IFLNK ) {
420 return BPC_FTYPE_SYMLINK;
421 } elsif ( ($mode & S_IFMT) == S_IFCHR ) {
422 return BPC_FTYPE_CHARDEV;
423 } elsif ( ($mode & S_IFMT) == S_IFBLK ) {
424 return BPC_FTYPE_BLOCKDEV;
425 } elsif ( ($mode & S_IFMT) == S_IFIFO ) {
426 return BPC_FTYPE_FIFO;
427 } elsif ( ($mode & S_IFMT) == S_IFSOCK ) {
428 return BPC_FTYPE_SOCKET;
430 return BPC_FTYPE_UNKNOWN;
435 # Set the attributes for a file. Returns non-zero on error.
439 my($fio, $f, $placeHolder) = @_;
442 if ( $f->{name} =~ m{(.*)/(.*)}s ) {
444 $dir = "$fio->{shareM}/" . $1;
445 } elsif ( $f->{name} eq "." ) {
447 $file = $fio->{share};
449 $dir = $fio->{shareM};
453 if ( !defined($fio->{attribLastDir}) || $fio->{attribLastDir} ne $dir ) {
455 # Flush any directories that don't match the first part
456 # of the new directory
458 foreach my $d ( keys(%{$fio->{attrib}}) ) {
459 next if ( $d eq "" || "$dir/" =~ m{^\Q$d/} );
460 $fio->attribWrite($d);
462 $fio->{attribLastDir} = $dir;
464 if ( !exists($fio->{attrib}{$dir}) ) {
465 $fio->{attrib}{$dir} = BackupPC::Attrib->new({
466 compress => $fio->{xfer}{compress},
468 my $path = $fio->{outDir} . $dir;
469 if ( -f $fio->{attrib}{$dir}->fileName($path)
470 && !$fio->{attrib}{$dir}->read($path) ) {
471 $fio->log(sprintf("Unable to read attribute file %s",
472 $fio->{attrib}{$dir}->fileName($path)));
475 $fio->log("attribSet(dir=$dir, file=$file)") if ( $fio->{logLevel} >= 4 );
477 my $mode = $f->{mode};
479 $mode |= S_HLINK_TARGET if ( $f->{hlink_self} );
480 $fio->{attrib}{$dir}->set($file, {
481 type => $fio->mode2type($f),
485 size => $placeHolder ? -1 : $f->{size},
486 mtime => $f->{mtime},
497 # Don't write attributes on 2nd phase - they're already
498 # taken care of during the first phase.
500 return if ( $fio->{phase} > 0 );
501 if ( !defined($d) ) {
503 # flush all entries (in reverse order)
505 foreach $d ( sort({$b cmp $a} keys(%{$fio->{attrib}})) ) {
506 $fio->attribWrite($d);
510 return if ( !defined($fio->{attrib}{$d}) );
512 # Set deleted files in the attributes. Any file in the view
513 # that doesn't have attributes is flagged as deleted for
514 # incremental dumps. All files sent by rsync have attributes
515 # temporarily set so we can do deletion detection. We also
516 # prune these temporary attributes.
522 $dir = $1 if ( $d =~ m{.+?/(.*)}s );
523 $fio->viewCacheDir(undef, $dir);
524 ##print("attribWrite $d,$dir\n");
525 ##$Data::Dumper::Indent = 1;
526 ##$fio->log("attribWrite $d,$dir");
527 ##$fio->log("viewCacheLogKeys = ", keys(%{$fio->{viewCache}}));
528 ##$fio->log("attribKeys = ", keys(%{$fio->{attrib}}));
529 ##print "viewCache = ", Dumper($fio->{attrib});
530 ##print "attrib = ", Dumper($fio->{attrib});
531 if ( defined($fio->{viewCache}{$d}) ) {
532 foreach my $f ( keys(%{$fio->{viewCache}{$d}}) ) {
534 $name = "$1/$name" if ( $d =~ m{.*?/(.*)}s );
535 if ( defined(my $a = $fio->{attrib}{$d}->get($f)) ) {
537 # delete temporary attributes (skipped files)
539 if ( $a->{size} < 0 ) {
540 $fio->{attrib}{$d}->set($f, undef);
541 $fio->logFileAction("skip", {
542 %{$fio->{viewCache}{$d}{$f}},
544 }) if ( $fio->{logLevel} >= 2
545 && $a->{type} == BPC_FTYPE_FILE );
547 } elsif ( !$fio->{full} ) {
548 ##print("Delete file $f\n");
549 $fio->logFileAction("delete", {
550 %{$fio->{viewCache}{$d}{$f}},
552 }) if ( $fio->{logLevel} >= 1 );
553 $fio->{attrib}{$d}->set($f, {
554 type => BPC_FTYPE_DELETED,
565 if ( $fio->{attrib}{$d}->fileCount ) {
566 my $data = $fio->{attrib}{$d}->writeData;
569 $dirM = $1 . "/" . $fio->{bpc}->fileNameMangle($2)
570 if ( $dirM =~ m{(.*?)/(.*)}s );
571 my $fileName = $fio->{attrib}{$d}->fileName("$fio->{outDir}$dirM");
572 $fio->log("attribWrite(dir=$d) -> $fileName")
573 if ( $fio->{logLevel} >= 4 );
574 my $poolWrite = BackupPC::PoolWrite->new($fio->{bpc}, $fileName,
575 length($data), $fio->{xfer}{compress});
576 $poolWrite->write(\$data);
577 $fio->processClose($poolWrite, $fio->{attrib}{$d}->fileName($dirM),
580 delete($fio->{attrib}{$d});
585 my($fio, $poolWrite, $fileName, $origSize, $doStats) = @_;
586 my($exists, $digest, $outSize, $errs) = $poolWrite->close;
588 $fileName =~ s{^/+}{};
589 $fio->log(@$errs) if ( defined($errs) && @$errs );
591 $fio->{stats}{TotalFileCnt}++;
592 $fio->{stats}{TotalFileSize} += $origSize;
596 $fio->{stats}{ExistFileCnt}++;
597 $fio->{stats}{ExistFileSize} += $origSize;
598 $fio->{stats}{ExistFileCompSize} += $outSize;
600 } elsif ( $outSize > 0 ) {
601 my $fh = $fio->{newFilesFH};
602 print($fh "$digest $origSize $fileName\n") if ( defined($fh) );
604 return $exists && $origSize > 0;
611 return $fio->{stats};
615 # Make a given directory. Returns non-zero on error.
620 my $name = $1 if ( $f->{name} =~ /(.*)/s );
623 if ( $name eq "." ) {
624 $path = $fio->{outDirSh};
626 $path = $fio->{outDirSh} . $fio->{bpc}->fileNameMangle($name);
628 $fio->logFileAction("create", $f) if ( $fio->{logLevel} >= 1 );
629 $fio->log("makePath($path, 0777)") if ( $fio->{logLevel} >= 5 );
630 $path = $1 if ( $path =~ /(.*)/s );
631 File::Path::mkpath($path, 0, 0777) if ( !-d $path );
632 return $fio->attribSet($f) if ( -d $path );
633 $fio->log("Can't create directory $path");
634 $fio->{stats}{errorCnt}++;
639 # Make a special file. Returns non-zero on error.
644 my $name = $1 if ( $f->{name} =~ /(.*)/s );
645 my $fNameM = $fio->{bpc}->fileNameMangle($name);
646 my $path = $fio->{outDirSh} . $fNameM;
647 my $attr = $fio->attribGet($f);
649 my $type = $fio->mode2type($f);
651 $fio->log("makeSpecial($path, $type, $f->{mode})")
652 if ( $fio->{logLevel} >= 5 );
653 if ( $type == BPC_FTYPE_CHARDEV || $type == BPC_FTYPE_BLOCKDEV ) {
654 my($major, $minor, $fh, $fileData);
656 if ( defined($f->{rdev_major}) ) {
657 $major = $f->{rdev_major};
658 $minor = $f->{rdev_minor};
660 $major = $f->{rdev} >> 8;
661 $minor = $f->{rdev} & 0xff;
663 $str = "$major,$minor";
664 } elsif ( ($f->{mode} & S_IFMT) == S_IFLNK ) {
666 } elsif ( ($f->{mode} & S_IFMT) == S_IFREG ) {
670 if ( !defined($f->{hlink}) ) {
671 $fio->log("Error: makeSpecial($path, $type, $f->{mode}) called"
672 . " on a regular non-hardlink file");
678 # Now see if the file is different, or this is a full, in which
679 # case we create the new file.
684 || $attr->{type} != $type
685 || $attr->{mtime} != $f->{mtime}
686 || $attr->{size} != $f->{size}
687 || $attr->{uid} != $f->{uid}
688 || $attr->{gid} != $f->{gid}
689 || $attr->{mode} != $f->{mode}
690 || $attr->{hlink_self} != $f->{hlink_self}
691 || !defined($fh = BackupPC::FileZIO->open($attr->{fullPath}, 0,
693 || $fh->read(\$fileData, length($str) + 1) != length($str)
694 || $fileData ne $str ) {
695 $fh->close if ( defined($fh) );
696 $fh = BackupPC::PoolWrite->new($fio->{bpc}, $path,
697 length($str), $fio->{xfer}{compress});
699 my $exist = $fio->processClose($fh, "$fio->{shareM}/$fNameM",
701 $fio->logFileAction($exist ? "pool" : "create", $f)
702 if ( $fio->{logLevel} >= 1 );
703 return $fio->attribSet($f);
705 $fio->logFileAction("skip", $f) if ( $fio->{logLevel} >= 2 );
707 $fh->close if ( defined($fh) );
711 # Make a hardlink. Returns non-zero on error.
712 # This actually gets called twice for each hardlink.
713 # Once as the file list is processed, and again at
714 # the end. BackupPC does them as it goes (since it is
715 # just saving the hardlink info and not actually making
720 my($fio, $f, $end) = @_;
723 return $fio->makeSpecial($f) if ( !$f->{hlink_self} );
728 my($fio, $path) = @_;
730 $fio->log("Unexpected call BackupPC::Xfer::RsyncFileIO->unlink($path)");
734 # Default log handler
740 print(STDERR $str, "\n");
744 # Handle one or more log messages
748 my($fio, @logStr) = @_;
750 foreach my $str ( @logStr ) {
751 next if ( $str eq "" );
752 $fio->{logHandler}($str);
757 # Generate a log file message for a completed file
761 my($fio, $action, $f) = @_;
762 my $owner = "$f->{uid}/$f->{gid}";
763 my $type = (("", "p", "c", "", "d", "", "b", "", "", "", "l", "", "s"))
764 [($f->{mode} & S_IFMT) >> 12];
765 my $name = $f->{name};
767 if ( ($f->{mode} & S_IFMT) == S_IFLNK ) {
768 $name .= " -> $f->{link}";
769 } elsif ( ($f->{mode} & S_IFMT) == S_IFREG
770 && defined($f->{hlink}) && !$f->{hlink_self} ) {
771 $name .= " -> $f->{hlink}";
775 $fio->log(sprintf(" %-6s %1s%4o %9s %11.0f %s",
785 # If there is a partial and we are doing a full, we do an incremental
786 # against the partial and a full against the rest. This subroutine
787 # is how we tell File::RsyncP which files to ignore attributes on
788 # (ie: against the partial dump we do consider the attributes, but
789 # otherwise we ignore attributes).
795 return if ( !defined($fio->{partialNum}) );
796 my($attr, $isPartial) = $fio->attribGetWhere($f);
797 $fio->log("$f->{name}: just checking attributes from partial")
798 if ( $isPartial && $fio->{logLevel} >= 5 );
803 # This is called by File::RsyncP when a file is skipped because the
808 my($fio, $f, $attr) = @_;
811 # Unless this is a partial, this is normal so ignore it.
813 return if ( !defined($fio->{partialNum}) );
815 $fio->log("$f->{name}: skipped in partial; adding link")
816 if ( $fio->{logLevel} >= 5 );
817 $fio->{rxLocalAttr} = $attr;
819 $fio->{rxSize} = $attr->{size};
820 delete($fio->{rxInFd});
821 delete($fio->{rxOutFd});
822 delete($fio->{rxDigest});
823 delete($fio->{rxInData});
824 return $fio->fileDeltaRxDone();
828 # Start receive of file deltas for a particular file.
832 my($fio, $f, $cnt, $size, $remainder) = @_;
834 $fio->{rxFile} = $f; # remote file attributes
835 $fio->{rxLocalAttr} = $fio->attribGet($f); # local file attributes
836 $fio->{rxBlkCnt} = $cnt; # how many blocks we will receive
837 $fio->{rxBlkSize} = $size; # block size
838 $fio->{rxRemainder} = $remainder; # size of the last block
839 $fio->{rxMatchBlk} = 0; # current start of match
840 $fio->{rxMatchNext} = 0; # current next block of match
841 $fio->{rxSize} = 0; # size of received file
842 my $rxSize = $cnt > 0 ? ($cnt - 1) * $size + $remainder : 0;
843 if ( $fio->{rxFile}{size} != $rxSize ) {
844 $fio->{rxMatchBlk} = undef; # size different, so no file match
845 $fio->log("$fio->{rxFile}{name}: size doesn't match"
846 . " ($fio->{rxFile}{size} vs $rxSize)")
847 if ( $fio->{logLevel} >= 5 );
850 # If compression was off and now on, or on and now off, then
851 # don't do an exact match.
853 if ( defined($fio->{rxLocalAttr})
854 && !$fio->{rxLocalAttr}{compress} != !$fio->{xfer}{compress} ) {
855 $fio->{rxMatchBlk} = undef; # compression changed, so no file match
856 $fio->log("$fio->{rxFile}{name}: compression changed, so no match"
857 . " ($fio->{rxLocalAttr}{compress} vs $fio->{xfer}{compress})")
858 if ( $fio->{logLevel} >= 4 );
861 # If the local file is a hardlink then no match
863 if ( defined($fio->{rxLocalAttr})
864 && $fio->{rxLocalAttr}{type} == BPC_FTYPE_HARDLINK ) {
865 $fio->{rxMatchBlk} = undef;
866 $fio->log("$fio->{rxFile}{name}: no match on hardlinks")
867 if ( $fio->{logLevel} >= 4 );
869 # need to copy since hardlink attribGet overwrites the name
871 $fio->{rxHLinkAttr} = $fio->attribGet($fCopy, 1); # hardlink attributes
873 delete($fio->{rxHLinkAttr});
875 delete($fio->{rxInFd});
876 delete($fio->{rxOutFd});
877 delete($fio->{rxDigest});
878 delete($fio->{rxInData});
882 # Process the next file delta for the current file. Returns 0 if ok,
883 # -1 if not. Must be called with either a block number, $blk, or new data,
884 # $newData, (not both) defined.
888 my($fio, $blk, $newData) = @_;
890 if ( defined($blk) ) {
891 if ( defined($fio->{rxMatchBlk}) && $fio->{rxMatchNext} == $blk ) {
893 # got the next block in order; just keep track.
895 $fio->{rxMatchNext}++;
899 my $newDataLen = length($newData);
900 $fio->log("$fio->{rxFile}{name}: blk=$blk, newData=$newDataLen, rxMatchBlk=$fio->{rxMatchBlk}, rxMatchNext=$fio->{rxMatchNext}")
901 if ( $fio->{logLevel} >= 8 );
902 if ( !defined($fio->{rxOutFd}) ) {
904 # maybe the file has no changes
906 if ( $fio->{rxMatchNext} == $fio->{rxBlkCnt}
907 && !defined($blk) && !defined($newData) ) {
908 #$fio->log("$fio->{rxFile}{name}: file is unchanged");
909 # if ( $fio->{logLevel} >= 8 );
914 # need to open an output file where we will build the
917 $fio->{rxFile}{name} =~ /(.*)/s;
918 my $rxOutFileRel = "$fio->{shareM}/" . $fio->{bpc}->fileNameMangle($1);
919 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
920 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
921 $rxOutFile, $fio->{rxFile}{size},
922 $fio->{xfer}{compress});
923 $fio->log("$fio->{rxFile}{name}: opening output file $rxOutFile")
924 if ( $fio->{logLevel} >= 9 );
925 $fio->{rxOutFile} = $rxOutFile;
926 $fio->{rxOutFileRel} = $rxOutFileRel;
927 $fio->{rxDigest} = File::RsyncP::Digest->new();
928 $fio->{rxDigest}->protocol($fio->{protocol_version});
929 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
931 if ( defined($fio->{rxMatchBlk})
932 && $fio->{rxMatchBlk} != $fio->{rxMatchNext} ) {
934 # Need to copy the sequence of blocks that matched. If the file
935 # is compressed we need to make a copy of the uncompressed file,
936 # since the compressed file is not seekable. Future optimizations
937 # could include only creating an uncompressed copy if the matching
938 # blocks were not monotonic, and to only do this if there are
939 # matching blocks (eg, maybe the entire file is new).
941 my $attr = $fio->{rxLocalAttr};
943 if ( !defined($fio->{rxInFd}) && !defined($fio->{rxInData}) ) {
944 my $inPath = $attr->{fullPath};
945 $inPath = $fio->{rxHLinkAttr}{fullPath}
946 if ( defined($fio->{rxHLinkAttr}) );
947 if ( $attr->{compress} ) {
948 if ( !defined($fh = BackupPC::FileZIO->open(
951 $attr->{compress})) ) {
952 $fio->log("Can't open $inPath");
953 $fio->{stats}{errorCnt}++;
956 if ( $attr->{size} < 16 * 1024 * 1024 ) {
958 # Cache the entire old file if it is less than 16MB
961 $fio->{rxInData} = "";
962 while ( $fh->read(\$data, 16 * 1024 * 1024) > 0 ) {
963 $fio->{rxInData} .= $data;
965 $fio->log("$attr->{fullPath}: cached all $attr->{size}"
967 if ( $fio->{logLevel} >= 9 );
970 # Create and write a temporary output file
972 unlink("$fio->{outDirSh}RStmp")
973 if ( -f "$fio->{outDirSh}RStmp" );
974 if ( open(F, "+>", "$fio->{outDirSh}RStmp") ) {
978 while ( $fh->read(\$data, 1024 * 1024) > 0 ) {
979 if ( syswrite(F, $data) != length($data) ) {
980 $fio->log(sprintf("Can't write len=%d to %s",
981 length($data) , "$fio->{outDirSh}RStmp"));
983 $fio->{stats}{errorCnt}++;
986 $byteCnt += length($data);
989 $fio->{rxInName} = "$fio->{outDirSh}RStmp";
990 sysseek($fio->{rxInFd}, 0, 0);
991 $fio->log("$attr->{fullPath}: copied $byteCnt,"
992 . "$attr->{size} bytes to $fio->{rxInName}")
993 if ( $fio->{logLevel} >= 9 );
995 $fio->log("Unable to open $fio->{outDirSh}RStmp");
997 $fio->{stats}{errorCnt}++;
1003 if ( open(F, "<", $inPath) ) {
1005 $fio->{rxInFd} = *F;
1006 $fio->{rxInName} = $attr->{fullPath};
1008 $fio->log("Unable to open $inPath");
1009 $fio->{stats}{errorCnt}++;
1014 my $lastBlk = $fio->{rxMatchNext} - 1;
1015 $fio->log("$fio->{rxFile}{name}: writing blocks $fio->{rxMatchBlk}.."
1017 if ( $fio->{logLevel} >= 9 );
1018 my $seekPosn = $fio->{rxMatchBlk} * $fio->{rxBlkSize};
1019 if ( defined($fio->{rxInFd})
1020 && !sysseek($fio->{rxInFd}, $seekPosn, 0) ) {
1021 $fio->log("Unable to seek $fio->{rxInName} to $seekPosn");
1022 $fio->{stats}{errorCnt}++;
1025 my $cnt = $fio->{rxMatchNext} - $fio->{rxMatchBlk};
1026 my($thisCnt, $len, $data);
1027 for ( my $i = 0 ; $i < $cnt ; $i += $thisCnt ) {
1028 $thisCnt = $cnt - $i;
1029 $thisCnt = 512 if ( $thisCnt > 512 );
1030 if ( $fio->{rxMatchBlk} + $i + $thisCnt == $fio->{rxBlkCnt} ) {
1031 $len = ($thisCnt - 1) * $fio->{rxBlkSize} + $fio->{rxRemainder};
1033 $len = $thisCnt * $fio->{rxBlkSize};
1035 if ( defined($fio->{rxInData}) ) {
1036 $data = substr($fio->{rxInData}, $seekPosn, $len);
1039 my $got = sysread($fio->{rxInFd}, $data, $len);
1040 if ( $got != $len ) {
1041 my $inFileSize = -s $fio->{rxInName};
1042 $fio->log("Unable to read $len bytes from $fio->{rxInName}"
1043 . " got=$got, seekPosn=$seekPosn"
1044 . " ($i,$thisCnt,$fio->{rxBlkCnt},$inFileSize"
1045 . ",$attr->{size})");
1046 $fio->{stats}{errorCnt}++;
1051 $fio->{rxOutFd}->write(\$data);
1052 $fio->{rxDigest}->add($data);
1053 $fio->{rxSize} += length($data);
1055 $fio->{rxMatchBlk} = undef;
1057 if ( defined($blk) ) {
1059 # Remember the new block number
1061 $fio->{rxMatchBlk} = $blk;
1062 $fio->{rxMatchNext} = $blk + 1;
1064 if ( defined($newData) ) {
1066 # Write the new chunk
1068 my $len = length($newData);
1069 $fio->log("$fio->{rxFile}{name}: writing $len bytes new data")
1070 if ( $fio->{logLevel} >= 9 );
1071 $fio->{rxOutFd}->write(\$newData);
1072 $fio->{rxDigest}->add($newData);
1073 $fio->{rxSize} += length($newData);
1078 # Finish up the current receive file. Returns undef if ok, -1 if not.
1079 # Returns 1 if the md4 digest doesn't match.
1083 my($fio, $md4, $phase) = @_;
1084 my $name = $1 if ( $fio->{rxFile}{name} =~ /(.*)/s );
1087 close($fio->{rxInFd}) if ( defined($fio->{rxInFd}) );
1088 unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" );
1089 $fio->{phase} = $phase;
1092 # Check the final md4 digest
1094 if ( defined($md4) ) {
1096 if ( !defined($fio->{rxDigest}) ) {
1098 # File was exact match, but we still need to verify the
1099 # MD4 checksum. Compute the md4 digest (or fetch the
1102 if ( defined(my $attr = $fio->{rxLocalAttr}) ) {
1104 # block size doesn't matter: we're only going to
1105 # fetch the md4 file digest, not the block digests.
1107 my($err, $csum, $blkSize)
1108 = BackupPC::Xfer::RsyncDigest->digestStart(
1109 $attr->{fullPath}, $attr->{size},
1110 0, 2048, $fio->{checksumSeed}, 1,
1111 $attr->{compress}, 1,
1112 $fio->{protocol_version});
1114 $fio->log("Can't open $attr->{fullPath} for MD4"
1115 . " check (err=$err, $name)");
1116 $fio->{stats}{errorCnt}++;
1118 if ( $fio->{logLevel} >= 5 ) {
1119 my($isCached, $invalid) = $csum->isCached;
1120 $fio->log("MD4 $attr->{fullPath} cache = $isCached,"
1121 . " invalid = $invalid");
1123 $newDigest = $csum->digestEnd;
1125 $fio->{rxSize} = $attr->{size};
1128 # Empty file; just create an empty file digest
1130 $fio->{rxDigest} = File::RsyncP::Digest->new();
1131 $fio->{rxDigest}->protocol($fio->{protocol_version});
1132 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
1133 $newDigest = $fio->{rxDigest}->digest;
1135 $fio->log("$name got exact match") if ( $fio->{logLevel} >= 5 );
1137 $newDigest = $fio->{rxDigest}->digest;
1139 if ( $fio->{logLevel} >= 3 ) {
1140 my $md4Str = unpack("H*", $md4);
1141 my $newStr = unpack("H*", $newDigest);
1142 $fio->log("$name got digests $md4Str vs $newStr")
1144 if ( $md4 ne $newDigest ) {
1146 $fio->log("$name: fatal error: md4 doesn't match on retry;"
1149 $fio->log("$name: md4 doesn't match: will retry in phase 1;"
1152 $fio->{stats}{errorCnt}++;
1153 if ( defined($fio->{rxOutFd}) ) {
1154 $fio->{rxOutFd}->close;
1155 unlink($fio->{rxOutFile});
1157 delete($fio->{rxFile});
1158 delete($fio->{rxOutFile});
1164 # One special case is an empty file: if the file size is
1165 # zero we need to open the output file to create it.
1167 if ( $fio->{rxSize} == 0 ) {
1168 my $rxOutFileRel = "$fio->{shareM}/"
1169 . $fio->{bpc}->fileNameMangle($name);
1170 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
1171 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
1172 $rxOutFile, $fio->{rxSize},
1173 $fio->{xfer}{compress});
1175 if ( !defined($fio->{rxOutFd}) ) {
1177 # No output file, meaning original was an exact match.
1179 $fio->log("$name: nothing to do")
1180 if ( $fio->{logLevel} >= 5 );
1181 my $attr = $fio->{rxLocalAttr};
1182 my $f = $fio->{rxFile};
1183 $fio->logFileAction("same", $f) if ( $fio->{logLevel} >= 1 );
1185 || $attr->{type} != $f->{type}
1186 || $attr->{mtime} != $f->{mtime}
1187 || $attr->{size} != $f->{size}
1188 || $attr->{uid} != $f->{uid}
1189 || $attr->{gid} != $f->{gid}
1190 || $attr->{mode} != $f->{mode}
1191 || $attr->{hlink_self} != $f->{hlink_self} ) {
1193 # In the full case, or if the attributes are different,
1194 # we need to make a link from the previous file and
1195 # set the attributes.
1197 my $rxOutFile = $fio->{outDirSh}
1198 . $fio->{bpc}->fileNameMangle($name);
1199 my($exists, $digest, $origSize, $outSize, $errs)
1200 = BackupPC::PoolWrite::LinkOrCopy(
1205 $fio->{xfer}{compress});
1207 # Cumulate the stats
1209 $fio->{stats}{TotalFileCnt}++;
1210 $fio->{stats}{TotalFileSize} += $fio->{rxSize};
1211 $fio->{stats}{ExistFileCnt}++;
1212 $fio->{stats}{ExistFileSize} += $fio->{rxSize};
1213 $fio->{stats}{ExistFileCompSize} += -s $rxOutFile;
1214 $fio->{rxFile}{size} = $fio->{rxSize};
1215 $ret = $fio->attribSet($fio->{rxFile});
1216 $fio->log(@$errs) if ( defined($errs) && @$errs );
1218 if ( !$exists && $outSize > 0 ) {
1220 # the hard link failed, most likely because the target
1221 # file has too many links. We have copied the file
1222 # instead, so add this to the new file list.
1224 my $rxOutFileRel = "$fio->{shareM}/"
1225 . $fio->{bpc}->fileNameMangle($name);
1226 $rxOutFileRel =~ s{^/+}{};
1227 my $fh = $fio->{newFilesFH};
1228 print($fh "$digest $origSize $rxOutFileRel\n")
1229 if ( defined($fh) );
1233 my $exist = $fio->processClose($fio->{rxOutFd},
1234 $fio->{rxOutFileRel},
1236 $fio->logFileAction($exist ? "pool" : "create", $fio->{rxFile})
1237 if ( $fio->{logLevel} >= 1 );
1238 $fio->{rxFile}{size} = $fio->{rxSize};
1239 $ret = $fio->attribSet($fio->{rxFile});
1241 delete($fio->{rxDigest});
1242 delete($fio->{rxInData});
1243 delete($fio->{rxFile});
1244 delete($fio->{rxOutFile});
1249 # Callback function for BackupPC::View->find. Note the order of the
1250 # first two arguments.
1254 my($a, $fio, $fList, $outputFunc) = @_;
1255 my $name = $a->{relPath};
1257 my $type = $a->{type};
1258 my $extraAttribs = {};
1260 if ( $a->{mode} & S_HLINK_TARGET ) {
1261 $a->{hlink_self} = 1;
1262 $a->{mode} &= ~S_HLINK_TARGET;
1264 $n =~ s/^\Q$fio->{xfer}{pathHdrSrc}//;
1265 $fio->log("Sending $name (remote=$n) type = $type") if ( $fio->{logLevel} >= 1 );
1266 if ( $type == BPC_FTYPE_CHARDEV
1267 || $type == BPC_FTYPE_BLOCKDEV
1268 || $type == BPC_FTYPE_SYMLINK ) {
1269 my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0, $a->{compress});
1271 if ( defined($fh) ) {
1272 $rdSize = $fh->read(\$str, $a->{size} + 1024);
1273 if ( $type == BPC_FTYPE_SYMLINK ) {
1275 # Reconstruct symbolic link
1277 $extraAttribs = { link => $str };
1278 if ( $rdSize != $a->{size} ) {
1280 $fio->log("$name: can't read exactly $a->{size} bytes");
1281 $fio->{stats}{errorCnt}++;
1283 } elsif ( $str =~ /(\d*),(\d*)/ ) {
1285 # Reconstruct char or block special major/minor device num
1287 # Note: char/block devices have $a->{size} = 0, so we
1288 # can't do an error check on $rdSize.
1291 rdev => $1 * 256 + $2,
1296 $fio->log("$name: unexpected special file contents $str");
1297 $fio->{stats}{errorCnt}++;
1302 $fio->log("$name: can't open");
1303 $fio->{stats}{errorCnt}++;
1305 } elsif ( $fio->{preserve_hard_links}
1306 && ($type == BPC_FTYPE_HARDLINK || $type == BPC_FTYPE_FILE)
1307 && ($type == BPC_FTYPE_HARDLINK
1308 || $fio->{protocol_version} < 27
1309 || $a->{hlink_self}) ) {
1311 # Fill in fake inode information so that the remote rsync
1312 # can correctly create hardlinks.
1314 $name =~ s/^\.?\/+//;
1315 my($target, $inode);
1317 if ( $type == BPC_FTYPE_HARDLINK ) {
1318 my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0,
1320 if ( defined($fh) ) {
1321 $fh->read(\$target, 65536);
1323 $target =~ s/^\.?\/+//;
1324 if ( defined($fio->{hlinkFile2Num}{$target}) ) {
1325 $inode = $fio->{hlinkFile2Num}{$target};
1327 $inode = $fio->{fileListCnt};
1328 $fio->{hlinkFile2Num}{$target} = $inode;
1331 $fio->log("$a->{fullPath}: can't open for hardlink");
1332 $fio->{stats}{errorCnt}++;
1334 } elsif ( $a->{hlink_self} ) {
1335 if ( defined($fio->{hlinkFile2Num}{$name}) ) {
1336 $inode = $fio->{hlinkFile2Num}{$name};
1338 $inode = $fio->{fileListCnt};
1339 $fio->{hlinkFile2Num}{$name} = $inode;
1342 $inode = $fio->{fileListCnt} if ( !defined($inode) );
1343 $fio->log("$name: setting inode to $inode");
1352 mode => $a->{mode} & ~S_HLINK_TARGET,
1355 mtime => $a->{mtime},
1359 my $logName = $f->{name};
1360 from_to($f->{name}, "utf8", $fio->{clientCharset})
1361 if ( $fio->{clientCharset} ne "" );
1364 $logName = "$fio->{xfer}{pathHdrDest}/$logName";
1365 $logName =~ s{//+}{/}g;
1366 $f->{name} = $logName;
1367 $fio->logFileAction("restore", $f) if ( $fio->{logLevel} >= 1 );
1369 &$outputFunc($fList->encodeData);
1373 $fio->{fileListCnt}++;
1374 if ( $type != BPC_FTYPE_DIR ) {
1375 $fio->{stats}{TotalFileCnt}++;
1376 $fio->{stats}{TotalFileSize} += $a->{size};
1382 my($fio, $flist, $outputFunc) = @_;
1385 # Populate the file list with the files requested by the user.
1386 # Since some might be directories so we call BackupPC::View::find.
1388 $fio->log("fileListSend: sending file list: "
1389 . join(" ", @{$fio->{fileList}})) if ( $fio->{logLevel} >= 4 );
1390 $fio->{fileListCnt} = 0;
1391 $fio->{hlinkFile2Num} = {};
1392 foreach my $name ( @{$fio->{fileList}} ) {
1393 $fio->{view}->find($fio->{xfer}{bkupSrcNum},
1394 $fio->{xfer}{bkupSrcShare},
1396 \&fileListEltSend, $fio, $flist, $outputFunc);
1402 my($fio, $isChild) = @_;
1405 # If we are aborting early, remove the last file since
1406 # it was not complete
1408 if ( $isChild && defined($fio->{rxFile}) ) {
1409 unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" );
1410 if ( defined($fio->{rxFile}) ) {
1411 unlink($fio->{rxOutFile});
1412 $fio->log("finish: removing in-process file $fio->{rxFile}{name}");
1417 # Flush the attributes if this is the child
1419 $fio->attribWrite(undef) if ( $isChild );
1425 # join('',@_), kill 0;