X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;ds=sidebyside;f=lib%2FBackupPC%2FXfer%2FRsyncFileIO.pm;h=f4853bbbb9ddf778036e1d8f02b97e2f64f2a700;hb=99b9efe77451800084c477ed2139100f830cb7db;hp=2fa3d27663f9c3198a6b0381a85782bc50ef8fbd;hpb=2c14784ad71874ec850d189060fe63d6eb9eba95;p=BackupPC.git diff --git a/lib/BackupPC/Xfer/RsyncFileIO.pm b/lib/BackupPC/Xfer/RsyncFileIO.pm index 2fa3d27..f4853bb 100644 --- a/lib/BackupPC/Xfer/RsyncFileIO.pm +++ b/lib/BackupPC/Xfer/RsyncFileIO.pm @@ -12,7 +12,7 @@ # #======================================================================== # -# Version 2.1.0_CVS, released 8 Feb 2004. +# Version 2.1.2, released 5 Sep 2005. # # See http://backuppc.sourceforge.net. # @@ -24,9 +24,8 @@ use strict; use File::Path; use BackupPC::Attrib qw(:all); use BackupPC::View; -use BackupPC::RsyncDigest; +use BackupPC::Xfer::RsyncDigest qw(:all); use BackupPC::PoolWrite; -use Data::Dumper; use constant S_IFMT => 0170000; # type of file use constant S_IFDIR => 0040000; # directory @@ -98,6 +97,7 @@ sub logHandlerSet { my($fio, $sub) = @_; $fio->{logHandler} = $sub; + BackupPC::Xfer::RsyncDigest->logHandlerSet($sub); } # @@ -105,16 +105,55 @@ sub logHandlerSet # sub csumStart { - my($fio, $f, $needMD4, $defBlkSize) = @_; + my($fio, $f, $needMD4, $defBlkSize, $phase) = @_; + $defBlkSize ||= $fio->{blockSize}; my $attr = $fio->attribGet($f); $fio->{file} = $f; $fio->csumEnd if ( defined($fio->{csum}) ); return -1 if ( $attr->{type} != BPC_FTYPE_FILE ); + # + # Rsync uses short checksums on the first phase. If the whole-file + # checksum fails, then the file is repeated with full checksums. + # So on phase 2 we verify the checksums if they are cached. + # + if ( ($phase > 0 || rand(1) < $fio->{cacheCheckProb}) + && $attr->{compress} + && $fio->{checksumSeed} == RSYNC_CSUMSEED_CACHE ) { + my($err, $d, $blkSize) = BackupPC::Xfer::RsyncDigest->digestStart( + $attr->{fullPath}, $attr->{size}, 0, + $defBlkSize, $fio->{checksumSeed}, + 0, $attr->{compress}, 0); + my($isCached, $isInvalid) = $d->isCached; + if ( $fio->{logLevel} >= 5 ) { + $fio->log("$attr->{fullPath} verify; cached = $isCached," + . " invalid = $isInvalid, phase = $phase"); + } + if ( $isCached || $isInvalid ) { + my $ret = BackupPC::Xfer::RsyncDigest->digestAdd( + $attr->{fullPath}, $blkSize, + $fio->{checksumSeed}, 1 # verify + ); + if ( $ret != 1 ) { + $fio->log("Bad cached digest for $attr->{fullPath} ($ret);" + . " fixed"); + $fio->{stats}{errorCnt}++; + } else { + $fio->log("$f->{name}: verified cached digest") + if ( $fio->{logLevel} >= 2 ); + } + } + $d->digestEnd; + } (my $err, $fio->{csum}, my $blkSize) - = BackupPC::RsyncDigest->digestStart($attr->{fullPath}, $attr->{size}, - 0, $defBlkSize, $fio->{checksumSeed}, $needMD4, - $attr->{compress}, 1); + = BackupPC::Xfer::RsyncDigest->digestStart($attr->{fullPath}, + $attr->{size}, 0, $defBlkSize, $fio->{checksumSeed}, + $needMD4, $attr->{compress}, 1); + if ( $fio->{logLevel} >= 5 ) { + my($isCached, $invalid) = $fio->{csum}->isCached; + $fio->log("$attr->{fullPath} cache = $isCached," + . " invalid = $invalid, phase = $phase"); + } if ( $err ) { $fio->log("Can't get rsync digests from $attr->{fullPath}" . " (err=$err, name=$f->{name})"); @@ -191,6 +230,10 @@ sub checksumSeed my($fio, $checksumSeed) = @_; $fio->{checksumSeed} = $checksumSeed; + $fio->log("Checksum caching enabled (checksumSeed = $checksumSeed)") + if ( $fio->{logLevel} >= 1 && $checksumSeed == RSYNC_CSUMSEED_CACHE ); + $fio->log("Checksum seed is $checksumSeed") + if ( $fio->{logLevel} >= 2 && $checksumSeed != RSYNC_CSUMSEED_CACHE ); } sub dirs @@ -639,6 +682,10 @@ sub attrSkippedFile $fio->{rxLocalAttr} = $attr; $fio->{rxFile} = $f; $fio->{rxSize} = $attr->{size}; + delete($fio->{rxInFd}); + delete($fio->{rxOutFd}); + delete($fio->{rxDigest}); + delete($fio->{rxInData}); return $fio->fileDeltaRxDone(); } @@ -664,6 +711,17 @@ sub fileDeltaRxStart . " ($fio->{rxFile}{size} vs $rxSize)") if ( $fio->{logLevel} >= 5 ); } + # + # If compression was off and now on, or on and now off, then + # don't do an exact match. + # + if ( defined($fio->{rxLocalAttr}) + && !$fio->{rxLocalAttr}{compress} != !$fio->{xfer}{compress} ) { + $fio->{rxMatchBlk} = undef; # compression changed, so no file match + $fio->log("$fio->{rxFile}{name}: compression changed, so no match" + . " ($fio->{rxLocalAttr}{compress} vs $fio->{xfer}{compress})") + if ( $fio->{logLevel} >= 4 ); + } delete($fio->{rxInFd}); delete($fio->{rxOutFd}); delete($fio->{rxDigest}); @@ -868,8 +926,9 @@ sub fileDeltaRxNext # sub fileDeltaRxDone { - my($fio, $md4) = @_; + my($fio, $md4, $phase) = @_; my $name = $1 if ( $fio->{rxFile}{name} =~ /(.*)/ ); + my $ret; close($fio->{rxInFd}) if ( defined($fio->{rxInFd}) ); unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" ); @@ -891,15 +950,20 @@ sub fileDeltaRxDone # fetch the md4 file digest, not the block digests. # my($err, $csum, $blkSize) - = BackupPC::RsyncDigest->digestStart( + = BackupPC::Xfer::RsyncDigest->digestStart( $attr->{fullPath}, $attr->{size}, 0, 2048, $fio->{checksumSeed}, 1, - $attr->{compress}); + $attr->{compress}, 1); if ( $err ) { $fio->log("Can't open $attr->{fullPath} for MD4" . " check (err=$err, $name)"); $fio->{stats}{errorCnt}++; } else { + if ( $fio->{logLevel} >= 5 ) { + my($isCached, $invalid) = $csum->isCached; + $fio->log("MD4 $attr->{fullPath} cache = $isCached," + . " invalid = $invalid"); + } $newDigest = $csum->digestEnd; } $fio->{rxSize} = $attr->{size}; @@ -921,12 +985,20 @@ sub fileDeltaRxDone $fio->log("$name got digests $md4Str vs $newStr") } if ( $md4 ne $newDigest ) { - $fio->log("$name: fatal error: md4 doesn't match"); + if ( $phase > 0 ) { + $fio->log("$name: fatal error: md4 doesn't match on retry;" + . " file removed"); + } else { + $fio->log("$name: md4 doesn't match: will retry in phase 1;" + . " file removed"); + } $fio->{stats}{errorCnt}++; if ( defined($fio->{rxOutFd}) ) { $fio->{rxOutFd}->close; unlink($fio->{rxOutFile}); } + delete($fio->{rxFile}); + delete($fio->{rxOutFile}); return 1; } } @@ -968,32 +1040,34 @@ sub fileDeltaRxDone if ( !link($attr->{fullPath}, $rxOutFile) ) { $fio->log("Unable to link $attr->{fullPath} to $rxOutFile"); $fio->{stats}{errorCnt}++; - return -1; - } - # - # Cumulate the stats - # - $fio->{stats}{TotalFileCnt}++; - $fio->{stats}{TotalFileSize} += $fio->{rxSize}; - $fio->{stats}{ExistFileCnt}++; - $fio->{stats}{ExistFileSize} += $fio->{rxSize}; - $fio->{stats}{ExistFileCompSize} += -s $rxOutFile; - $fio->{rxFile}{size} = $fio->{rxSize}; - return $fio->attribSet($fio->{rxFile}); + $ret = -1; + } else { + # + # Cumulate the stats + # + $fio->{stats}{TotalFileCnt}++; + $fio->{stats}{TotalFileSize} += $fio->{rxSize}; + $fio->{stats}{ExistFileCnt}++; + $fio->{stats}{ExistFileSize} += $fio->{rxSize}; + $fio->{stats}{ExistFileCompSize} += -s $rxOutFile; + $fio->{rxFile}{size} = $fio->{rxSize}; + $ret = $fio->attribSet($fio->{rxFile}); + } } - } - if ( defined($fio->{rxOutFd}) ) { + } else { my $exist = $fio->processClose($fio->{rxOutFd}, $fio->{rxOutFileRel}, $fio->{rxSize}, 1); $fio->logFileAction($exist ? "pool" : "create", $fio->{rxFile}) if ( $fio->{logLevel} >= 1 ); $fio->{rxFile}{size} = $fio->{rxSize}; - return $fio->attribSet($fio->{rxFile}); + $ret = $fio->attribSet($fio->{rxFile}); } delete($fio->{rxDigest}); delete($fio->{rxInData}); - return; + delete($fio->{rxFile}); + delete($fio->{rxOutFile}); + return $ret; } # @@ -1093,10 +1167,22 @@ sub finish { my($fio, $isChild) = @_; + # + # If we are aborting early, remove the last file since + # it was not complete + # + if ( $isChild && defined($fio->{rxFile}) ) { + unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" ); + if ( defined($fio->{rxFile}) ) { + unlink($fio->{rxOutFile}); + $fio->log("finish: removing in-process file $fio->{rxFile}{name}"); + } + } + # # Flush the attributes if this is the child # - $fio->attribWrite(undef); + $fio->attribWrite(undef) if ( $isChild ); } #sub is_tainted