X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=lib%2FBackupPC%2FPoolWrite.pm;h=d0be0e653b4ecdeea228a85c37da1d3c2090fab3;hp=7957f2c0db41e869aa4ec5a10e460a72e26f724e;hb=e951f787a66c5bd9e9955c3f657a5b44289c0fe1;hpb=0697368bbcef14908cd4684cf07744dc840464de diff --git a/lib/BackupPC/PoolWrite.pm b/lib/BackupPC/PoolWrite.pm index 7957f2c..d0be0e6 100644 --- a/lib/BackupPC/PoolWrite.pm +++ b/lib/BackupPC/PoolWrite.pm @@ -38,7 +38,7 @@ # Craig Barratt # # COPYRIGHT -# Copyright (C) 2001 Craig Barratt +# Copyright (C) 2001-2003 Craig Barratt # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -56,7 +56,7 @@ # #======================================================================== # -# Version 2.0.0_CVS, released 18 Jan 2003. +# Version 2.1.0, released 20 Jun 2004. # # See http://backuppc.sourceforge.net. # @@ -89,6 +89,8 @@ sub new eof => undef, }, $class; + $self->{hardLinkMax} = $bpc->ConfValue("HardLinkMax"); + # # Always unlink any current file in case it is already linked # @@ -96,8 +98,8 @@ sub new return $self; } -my $BufSize = 1048576; # 1MB or 2^20 -my $MaxFiles = 20; +my $BufSize = 1048576; # 1MB or 2^20 +my $MaxFiles = 20; # max number of compare files open at one time sub write { @@ -106,11 +108,29 @@ sub write return if ( $a->{eof} ); $a->{data} .= $$dataRef if ( defined($dataRef) ); return if ( length($a->{data}) < $BufSize && defined($dataRef) ); - if ( !defined($a->{digest}) && $a->{fileSize} > 0 ) { + + # + # Correct the fileSize if it is wrong (rsync might transfer + # a file whose length is different to the length sent with the + # file list if the file changes between the file list sending + # and the file sending). Here we only catch the case where + # we haven't computed the digest (ie: we have written no more + # than $BufSize). We catch the big file case below. + # + if ( !defined($dataRef) && !defined($a->{digest}) + && $a->{fileSize} != length($a->{data}) ) { + #my $newSize = length($a->{data}); + #print("Fixing file size from $a->{fileSize} to $newSize\n"); + $a->{fileSize} = length($a->{data}); + } + + if ( !defined($a->{digest}) && length($a->{data}) > 0 ) { # # build a list of all the candidate matching files # my $md5 = Digest::MD5->new; + $a->{fileSize} = length($a->{data}) + if ( $a->{fileSize} < length($a->{data}) ); $a->{digest} = $a->{bpc}->Buffer2MD5($md5, $a->{fileSize}, \$a->{data}); if ( !defined($a->{base} = $a->{bpc}->MD52Path($a->{digest}, $a->{compress})) ) { @@ -122,7 +142,8 @@ sub write my $fileName = $a->{fileCnt} < 0 ? $a->{base} : "$a->{base}_$a->{fileCnt}"; last if ( !-f $fileName ); - if ( !defined($fh = BackupPC::FileZIO->open($fileName, 0, + if ( (stat(_))[3] >= $a->{hardLinkMax} + || !defined($fh = BackupPC::FileZIO->open($fileName, 0, $a->{compress})) ) { $a->{fileCnt}++; next; @@ -148,7 +169,7 @@ sub write } } my $dataLen = length($a->{data}); - if ( !defined($a->{fhOut}) && $a->{fileSize} > 0 ) { + if ( !defined($a->{fhOut}) && length($a->{data}) > 0 ) { # # See if the new chunk of data continues to match the # candidate files. @@ -176,7 +197,8 @@ sub write # while ( -f $fileName ) { my $fh; - if ( !defined($fh = BackupPC::FileZIO->open($fileName, 0, + if ( (stat(_))[3] >= $a->{hardLinkMax} + || !defined($fh = BackupPC::FileZIO->open($fileName, 0, $a->{compress})) ) { $a->{fileCnt}++; #print(" Discarding $fileName (open failed)\n"); @@ -250,9 +272,93 @@ sub write # We are at EOF, so finish up # $a->{eof} = 1; + + # + # Make sure the fileSize was correct. See above for comments about + # rsync. + # + if ( $a->{nWrite} != $a->{fileSize} ) { + # + # Oops, fileSize was wrong, so our MD5 digest was wrong and our + # effort to match files likely failed. This is ugly, but our + # only choice at this point is to re-write the entire file with + # the correct length. We need to rename the file, open it for + # reading, and then re-write the file with the correct length. + # + + #print("Doing big file fixup ($a->{fileSize} != $a->{nWrite})\n"); + + my($fh, $fileName); + $a->{fileSize} = $a->{nWrite}; + + if ( defined($a->{fhOut}) ) { + if ( $a->{fileName} =~ /(.*)\// ) { + $fileName = $1; + } else { + $fileName = "."; + } + # + # Find a unique target temporary file name + # + my $i = 0; + while ( -f "$fileName/t$$.$i" ) { + $i++; + } + $fileName = "$fileName/t$$.$i"; + $a->{fhOut}->close(); + if ( !rename($a->{fileName}, $fileName) + || !defined($fh = BackupPC::FileZIO->open($fileName, 0, + $a->{compress})) ) { + push(@{$a->{errors}}, "Can't rename $a->{fileName} -> $fileName" + . " or open during size fixup\n"); + } + #print("Using temporary name $fileName\n"); + } elsif ( defined($a->{files}) && defined($a->{files}[0]) ) { + # + # We haven't written anything yet, so just use the + # compare file to copy from. + # + $fh = $a->{files}[0]->{fh}; + $fh->rewind; + #print("Using compare file $a->{files}[0]->{name}\n"); + } + if ( defined($fh) ) { + my $poolWrite = BackupPC::PoolWrite->new($a->{bpc}, $a->{fileName}, + $a->{fileSize}, $a->{compress}); + my $nRead = 0; + + while ( $nRead < $a->{fileSize} ) { + my $thisRead = $a->{fileSize} - $nRead < $BufSize + ? $a->{fileSize} - $nRead : $BufSize; + my $data; + my $n = $fh->read(\$data, $thisRead); + if ( $n != $thisRead ) { + push(@{$a->{errors}}, + "Unable to read $thisRead bytes during resize" + . " from temp $fileName (got $n)\n"); + last; + } + $poolWrite->write(\$data); + $nRead += $thisRead; + } + $fh->close; + unlink($fileName) if ( defined($fileName) ); + if ( @{$a->{errors}} ) { + $poolWrite->close; + return (0, $a->{digest}, -s $a->{fileName}, $a->{errors}); + } else { + return $poolWrite->close; + } + } + } + + # + # Close the compare files + # foreach my $f ( @{$a->{files}} ) { $f->{fh}->close(); } + if ( $a->{fileSize} == 0 ) { # # Simply create an empty file @@ -273,12 +379,16 @@ sub write push(@{$a->{errors}}, "Botch, no matches on $a->{fileName}" . " ($a->{digest})\n"); } elsif ( @{$a->{files}} > 1 ) { - my $str = "Unexpected multiple matches on" - . " $a->{fileName} ($a->{digest})\n"; - for ( my $i = 0 ; $i < @{$a->{files}} ; $i++ ) { - $str .= " -> $a->{files}[$i]->{name}\n"; - } - push(@{$a->{errors}}, $str); + # + # This is no longer a real error because $Conf{HardLinkMax} + # could be hit, thereby creating identical pool files + # + #my $str = "Unexpected multiple matches on" + # . " $a->{fileName} ($a->{digest})\n"; + #for ( my $i = 0 ; $i < @{$a->{files}} ; $i++ ) { + # $str .= " -> $a->{files}[$i]->{name}\n"; + #} + #push(@{$a->{errors}}, $str); } #print(" Linking $a->{fileName} to $a->{files}[0]->{name}\n"); if ( @{$a->{files}} && !link($a->{files}[0]->{name}, $a->{fileName}) ) { @@ -302,6 +412,23 @@ sub close return $a->write(undef); } +# +# Abort a pool write +# +sub abort +{ + my($a) = @_; + + if ( defined($a->{fhOut}) ) { + $a->{fhOut}->close(); + unlink($a->{fileName}); + } + foreach my $f ( @{$a->{files}} ) { + $f->{fh}->close(); + } + $a->{files} = []; +} + # # Copy $nBytes from files $fhIn to $fhOut. #