X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=lib%2FBackupPC%2FXfer%2FRsyncFileIO.pm;h=cc29e293393ded465678b159612b35a157512faa;hp=44a61ad61e8ed77b501d79893c0f199430c9a6ae;hb=329e870f56fb6572fa697998d33676588034c149;hpb=0697368bbcef14908cd4684cf07744dc840464de diff --git a/lib/BackupPC/Xfer/RsyncFileIO.pm b/lib/BackupPC/Xfer/RsyncFileIO.pm index 44a61ad..cc29e29 100644 --- a/lib/BackupPC/Xfer/RsyncFileIO.pm +++ b/lib/BackupPC/Xfer/RsyncFileIO.pm @@ -8,11 +8,11 @@ # Craig Barratt # # COPYRIGHT -# Copyright (C) 2002 Craig Barratt +# Copyright (C) 2002-2003 Craig Barratt # #======================================================================== # -# Version 2.0.0_CVS, released 18 Jan 2003. +# Version 2.1.0_CVS, released 3 Jul 2003. # # See http://backuppc.sourceforge.net. # @@ -65,6 +65,7 @@ sub new attrib => {}, logHandler => \&logHandler, stats => { + errorCnt => 0, TotalFileCnt => 0, TotalFileSize => 0, ExistFileCnt => 0, @@ -113,6 +114,7 @@ sub csumStart 0, $attr->{compress})) ) { $fio->log("Can't open $attr->{fullPath} (name=$f->{name})"); + $fio->{stats}{errorCnt}++; return -1; } if ( $needMD4) { @@ -121,7 +123,6 @@ sub csumStart } else { delete($fio->{csumDigest}); } - alarm($fio->{timeout}) if ( defined($fio->{timeout}) ); } sub csumGet @@ -134,14 +135,16 @@ sub csumGet return if ( !defined($fio->{fh}) ); if ( $fio->{fh}->read(\$fileData, $blockSize * $num) <= 0 ) { - return; + $fio->log("$fio->{file}{name}: csumGet is at EOF - zero padding"); + $fio->{stats}{errorCnt}++; + $fileData = pack("c", 0) x ($blockSize * $num); } $fio->{csumDigest}->add($fileData) if ( defined($fio->{csumDigest}) ); - $fio->log(sprintf("%s: getting csum ($num,$csumLen,%d,0x%x)\n", + $fio->log(sprintf("%s: getting csum ($num,$csumLen,%d,0x%x)", $fio->{file}{name}, length($fileData), $fio->{checksumSeed})) - if ( $fio->{logLevel} >= 10 ); + if ( $fio->{logLevel} >= 9 ); return $fio->{digest}->blockDigest($fileData, $blockSize, $csumLen, $fio->{checksumSeed}); } @@ -176,10 +179,10 @@ sub readStart 0, $attr->{compress})) ) { $fio->log("Can't open $attr->{fullPath} (name=$f->{name})"); + $fio->{stats}{errorCnt}++; return; } $fio->log("$f->{name}: opened for read") if ( $fio->{logLevel} >= 4 ); - alarm($fio->{timeout}) if ( defined($fio->{timeout}) ); } sub read @@ -434,7 +437,7 @@ sub attribWrite my $poolWrite = BackupPC::PoolWrite->new($fio->{bpc}, $fileName, length($data), $fio->{xfer}{compress}); $poolWrite->write(\$data); - $fio->processClose($poolWrite, $fio->{attrib}{$d}->fileName($d), + $fio->processClose($poolWrite, $fio->{attrib}{$d}->fileName($dirM), length($data), 0); } delete($fio->{attrib}{$d}); @@ -491,6 +494,7 @@ sub makePath File::Path::mkpath($path, 0, 0777) if ( !-d $path ); return $fio->attribSet($f) if ( -d $path ); $fio->log("Can't create directory $path"); + $fio->{stats}{errorCnt}++; return -1; } @@ -623,7 +627,8 @@ sub fileDeltaRxStart $fio->{rxRemainder} = $remainder; # size of the last block $fio->{rxMatchBlk} = 0; # current start of match $fio->{rxMatchNext} = 0; # current next block of match - my $rxSize = ($cnt - 1) * $size + $remainder; + $fio->{rxSize} = 0; # size of received file + my $rxSize = $cnt > 0 ? ($cnt - 1) * $size + $remainder : 0; if ( $fio->{rxFile}{size} != $rxSize ) { $fio->{rxMatchBlk} = undef; # size different, so no file match $fio->log("$fio->{rxFile}{name}: size doesn't match" @@ -634,7 +639,6 @@ sub fileDeltaRxStart delete($fio->{rxOutFd}); delete($fio->{rxDigest}); delete($fio->{rxInData}); - alarm($fio->{timeout}) if ( defined($fio->{timeout}) ); } # @@ -680,7 +684,7 @@ sub fileDeltaRxNext $rxOutFile, $fio->{rxFile}{size}, $fio->{xfer}{compress}); $fio->log("$fio->{rxFile}{name}: opening output file $rxOutFile") - if ( $fio->{logLevel} >= 10 ); + if ( $fio->{logLevel} >= 9 ); $fio->{rxOutFile} = $rxOutFile; $fio->{rxOutFileRel} = $rxOutFileRel; $fio->{rxDigest} = File::RsyncP::Digest->new; @@ -706,49 +710,63 @@ sub fileDeltaRxNext 0, $attr->{compress})) ) { $fio->log("Can't open $attr->{fullPath}"); + $fio->{stats}{errorCnt}++; return -1; } - if ( $attr->{size} < 10 * 1024 * 1024 ) { + if ( $attr->{size} < 16 * 1024 * 1024 ) { # - # Cache the entire old file if it is less than 10MB + # Cache the entire old file if it is less than 16MB # my $data; $fio->{rxInData} = ""; - while ( $fh->read(\$data, 10 * 1024 * 1024) > 0 ) { + while ( $fh->read(\$data, 16 * 1024 * 1024) > 0 ) { $fio->{rxInData} .= $data; } + $fio->log("$attr->{fullPath}: cached all $attr->{size}" + . " bytes") + if ( $fio->{logLevel} >= 9 ); } else { # # Create and write a temporary output file # unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" ); - if ( open(F, ">+", "$fio->{outDirSh}RStmp") ) { + if ( open(F, "+>", "$fio->{outDirSh}RStmp") ) { my $data; + my $byteCnt = 0; + binmode(F); while ( $fh->read(\$data, 1024 * 1024) > 0 ) { if ( syswrite(F, $data) != length($data) ) { $fio->log(sprintf("Can't write len=%d to %s", length($data) , "$fio->{outDirSh}RStmp")); $fh->close; + $fio->{stats}{errorCnt}++; return -1; } + $byteCnt += length($data); } $fio->{rxInFd} = *F; $fio->{rxInName} = "$fio->{outDirSh}RStmp"; - seek($fio->{rxInFd}, 0, 0); + sysseek($fio->{rxInFd}, 0, 0); + $fio->log("$attr->{fullPath}: copied $byteCnt," + . "$attr->{size} bytes to $fio->{rxInName}") + if ( $fio->{logLevel} >= 9 ); } else { $fio->log("Unable to open $fio->{outDirSh}RStmp"); $fh->close; + $fio->{stats}{errorCnt}++; return -1; } } $fh->close; } else { if ( open(F, "<", $attr->{fullPath}) ) { + binmode(F); $fio->{rxInFd} = *F; $fio->{rxInName} = $attr->{fullPath}; } else { $fio->log("Unable to open $attr->{fullPath}"); + $fio->{stats}{errorCnt}++; return -1; } } @@ -756,10 +774,12 @@ sub fileDeltaRxNext my $lastBlk = $fio->{rxMatchNext} - 1; $fio->log("$fio->{rxFile}{name}: writing blocks $fio->{rxMatchBlk}.." . "$lastBlk") - if ( $fio->{logLevel} >= 10 ); + if ( $fio->{logLevel} >= 9 ); my $seekPosn = $fio->{rxMatchBlk} * $fio->{rxBlkSize}; - if ( defined($fio->{rxInFd}) && !seek($fio->{rxInFd}, $seekPosn, 0) ) { - $fio->log("Unable to seek $attr->{fullPath} to $seekPosn"); + if ( defined($fio->{rxInFd}) + && !sysseek($fio->{rxInFd}, $seekPosn, 0) ) { + $fio->log("Unable to seek $attr->{rxInName} to $seekPosn"); + $fio->{stats}{errorCnt}++; return -1; } my $cnt = $fio->{rxMatchNext} - $fio->{rxMatchBlk}; @@ -774,16 +794,23 @@ sub fileDeltaRxNext } if ( defined($fio->{rxInData}) ) { $data = substr($fio->{rxInData}, $seekPosn, $len); + $seekPosn += $len; } else { - if ( sysread($fio->{rxInFd}, $data, $len) != $len ) { - $fio->log("Unable to read $len bytes from" - . " $fio->{rxInName} " - . "($i,$thisCnt,$fio->{rxBlkCnt})"); + my $got = sysread($fio->{rxInFd}, $data, $len); + if ( $got != $len ) { + my $inFileSize = -s $fio->{rxInName}; + $fio->log("Unable to read $len bytes from $fio->{rxInName}" + . " got=$got, seekPosn=$seekPosn" + . " ($i,$thisCnt,$fio->{rxBlkCnt},$inFileSize" + . ",$attr->{size})"); + $fio->{stats}{errorCnt}++; return -1; } + $seekPosn += $len; } $fio->{rxOutFd}->write(\$data); $fio->{rxDigest}->add($data); + $fio->{rxSize} += length($data); } $fio->{rxMatchBlk} = undef; } @@ -800,9 +827,10 @@ sub fileDeltaRxNext # my $len = length($newData); $fio->log("$fio->{rxFile}{name}: writing $len bytes new data") - if ( $fio->{logLevel} >= 10 ); + if ( $fio->{logLevel} >= 9 ); $fio->{rxOutFd}->write(\$newData); $fio->{rxDigest}->add($newData); + $fio->{rxSize} += length($newData); } } @@ -823,17 +851,21 @@ sub fileDeltaRxDone $fio->{rxDigest} = File::RsyncP::Digest->new; $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed})); my $attr = $fio->{rxLocalAttr}; - if ( defined($attr) && defined(my $fh = BackupPC::FileZIO->open( + if ( defined($attr) ) { + if ( defined(my $fh = BackupPC::FileZIO->open( $attr->{fullPath}, 0, $attr->{compress})) ) { - my $data; - while ( $fh->read(\$data, 4 * 65536) > 0 ) { - $fio->{rxDigest}->add($data); + my $data; + while ( $fh->read(\$data, 4 * 65536) > 0 ) { + $fio->{rxDigest}->add($data); + $fio->{rxSize} += length($data); + } + $fh->close; + } else { + $fio->log("Can't open $attr->{fullPath} for MD4 check ($name)"); + $fio->{stats}{errorCnt}++; } - $fh->close; - } else { - # ERROR } $fio->log("$name got exact match") if ( $fio->{logLevel} >= 5 ); @@ -847,8 +879,8 @@ sub fileDeltaRxDone $fio->log("$name got digests $md4Str vs $newStr") } if ( $md4 ne $newDigest ) { - $fio->log("$name md4 doesn't match") - if ( $fio->{logLevel} >= 1 ); + $fio->log("$name: fatal error: md4 doesn't match"); + $fio->{stats}{errorCnt}++; if ( defined($fio->{rxOutFd}) ) { $fio->{rxOutFd}->close; unlink($fio->{rxOutFile}); @@ -859,12 +891,12 @@ sub fileDeltaRxDone # One special case is an empty file: if the file size is # zero we need to open the output file to create it. # - if ( $fio->{rxFile}{size} == 0 ) { + if ( $fio->{rxSize} == 0 ) { my $rxOutFileRel = "$fio->{shareM}/" . $fio->{bpc}->fileNameMangle($name); my $rxOutFile = $fio->{outDir} . $rxOutFileRel; $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc}, - $rxOutFile, $fio->{rxFile}{size}, + $rxOutFile, $fio->{rxSize}, $fio->{xfer}{compress}); } if ( !defined($fio->{rxOutFd}) ) { @@ -891,25 +923,29 @@ sub fileDeltaRxDone . $fio->{bpc}->fileNameMangle($name); if ( !link($attr->{fullPath}, $rxOutFile) ) { $fio->log("Unable to link $attr->{fullPath} to $rxOutFile"); + $fio->{stats}{errorCnt}++; return -1; } # # Cumulate the stats # $fio->{stats}{TotalFileCnt}++; - $fio->{stats}{TotalFileSize} += $fio->{rxFile}{size}; + $fio->{stats}{TotalFileSize} += $fio->{rxSize}; $fio->{stats}{ExistFileCnt}++; - $fio->{stats}{ExistFileSize} += $fio->{rxFile}{size}; + $fio->{stats}{ExistFileSize} += $fio->{rxSize}; $fio->{stats}{ExistFileCompSize} += -s $rxOutFile; - return; + $fio->{rxFile}{size} = $fio->{rxSize}; + return $fio->attribSet($fio->{rxFile}); } } if ( defined($fio->{rxOutFd}) ) { my $exist = $fio->processClose($fio->{rxOutFd}, $fio->{rxOutFileRel}, - $fio->{rxFile}{size}, 1); + $fio->{rxSize}, 1); $fio->logFileAction($exist ? "pool" : "create", $fio->{rxFile}) if ( $fio->{logLevel} >= 1 ); + $fio->{rxFile}{size} = $fio->{rxSize}; + return $fio->attribSet($fio->{rxFile}); } delete($fio->{rxDigest}); delete($fio->{rxInData}); @@ -934,31 +970,36 @@ sub fileListEltSend || $type == BPC_FTYPE_BLOCKDEV || $type == BPC_FTYPE_SYMLINK ) { my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0, $a->{compress}); - my $str; + my($str, $rdSize); if ( defined($fh) ) { - if ( $fh->read(\$str, $a->{size} + 1) == $a->{size} ) { - if ( $type == BPC_FTYPE_SYMLINK ) { - # - # Reconstruct symbolic link - # - $extraAttribs = { link => $str }; - } elsif ( $str =~ /(\d*),(\d*)/ ) { - # - # Reconstruct char or block special major/minor device num - # - $extraAttribs = { rdev => $1 * 256 + $2 }; - } else { + $rdSize = $fh->read(\$str, $a->{size} + 1024); + if ( $type == BPC_FTYPE_SYMLINK ) { + # + # Reconstruct symbolic link + # + $extraAttribs = { link => $str }; + if ( $rdSize != $a->{size} ) { # ERROR - $fio->log("$name: unexpected file contents $str"); + $fio->log("$name: can't read exactly $a->{size} bytes"); + $fio->{stats}{errorCnt}++; } + } elsif ( $str =~ /(\d*),(\d*)/ ) { + # + # Reconstruct char or block special major/minor device num + # + # Note: char/block devices have $a->{size} = 0, so we + # can't do an error check on $rdSize. + # + $extraAttribs = { rdev => $1 * 256 + $2 }; } else { - # ERROR - $fio->log("$name: can't read exactly $a->{size} bytes"); + $fio->log("$name: unexpected special file contents $str"); + $fio->{stats}{errorCnt}++; } $fh->close; } else { # ERROR $fio->log("$name: can't open"); + $fio->{stats}{errorCnt}++; } } my $f = { @@ -984,7 +1025,6 @@ sub fileListEltSend $fio->{stats}{TotalFileCnt}++; $fio->{stats}{TotalFileSize} += $a->{size}; } - alarm($fio->{timeout}) if ( defined($fio->{timeout}) ); } sub fileListSend @@ -1013,7 +1053,6 @@ sub finish # Flush the attributes if this is the child # $fio->attribWrite(undef); - alarm($fio->{timeout}) if ( defined($fio->{timeout}) ); } #sub is_tainted