X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=lib%2FBackupPC%2FXfer%2FRsyncFileIO.pm;h=83b9db968f08f8d7b1c7f1a3994813141c6123cd;hp=b028b65682fcda43ee4db2ae8aee96f542748749;hb=f6fbcc3682d2bc9e7dfdc26e95bd5fcdb359496d;hpb=5c6a6cc4f333ce44a9df62ab828b0b9341579f7c diff --git a/lib/BackupPC/Xfer/RsyncFileIO.pm b/lib/BackupPC/Xfer/RsyncFileIO.pm index b028b65..83b9db9 100644 --- a/lib/BackupPC/Xfer/RsyncFileIO.pm +++ b/lib/BackupPC/Xfer/RsyncFileIO.pm @@ -8,11 +8,11 @@ # Craig Barratt # # COPYRIGHT -# Copyright (C) 2002-2003 Craig Barratt +# Copyright (C) 2002-2007 Craig Barratt # #======================================================================== # -# Version 2.1.0beta1, released 9 Apr 2004. +# Version 3.2.0beta0, released 5 April 2009. # # See http://backuppc.sourceforge.net. # @@ -22,19 +22,21 @@ package BackupPC::Xfer::RsyncFileIO; use strict; use File::Path; +use Encode qw/from_to/; use BackupPC::Attrib qw(:all); use BackupPC::View; -use BackupPC::Xfer::RsyncDigest; +use BackupPC::Xfer::RsyncDigest qw(:all); use BackupPC::PoolWrite; -use constant S_IFMT => 0170000; # type of file -use constant S_IFDIR => 0040000; # directory -use constant S_IFCHR => 0020000; # character special -use constant S_IFBLK => 0060000; # block special -use constant S_IFREG => 0100000; # regular -use constant S_IFLNK => 0120000; # symbolic link -use constant S_IFSOCK => 0140000; # socket -use constant S_IFIFO => 0010000; # fifo +use constant S_HLINK_TARGET => 0400000; # this file is hardlink target +use constant S_IFMT => 0170000; # type of file +use constant S_IFDIR => 0040000; # directory +use constant S_IFCHR => 0020000; # character special +use constant S_IFBLK => 0060000; # block special +use constant S_IFREG => 0100000; # regular +use constant S_IFLNK => 0120000; # symbolic link +use constant S_IFSOCK => 0140000; # socket +use constant S_IFIFO => 0010000; # fifo use vars qw( $RsyncLibOK ); @@ -59,7 +61,7 @@ sub new my $fio = bless { blockSize => 700, logLevel => 0, - digest => File::RsyncP::Digest->new, + digest => File::RsyncP::Digest->new(), checksumSeed => 0, attrib => {}, logHandler => \&logHandler, @@ -74,6 +76,7 @@ sub new %$options, }, $class; + $fio->{digest}->protocol($fio->{protocol_version}); $fio->{shareM} = $fio->{bpc}->fileNameEltMangle($fio->{share}); $fio->{outDir} = "$fio->{xfer}{outDir}/new/"; $fio->{outDirSh} = "$fio->{outDir}/$fio->{shareM}/"; @@ -85,6 +88,20 @@ sub new return $fio; } +# +# We publish our version to File::RsyncP. This is so File::RsyncP +# can provide backward compatibility to older FileIO code. +# +# Versions: +# +# undef or 1: protocol version 26, no hardlinks +# 2: protocol version 28, supports hardlinks +# +sub version +{ + return 2; +} + sub blockSize { my($fio, $value) = @_; @@ -93,10 +110,30 @@ sub blockSize return $fio->{blockSize}; } +sub protocol_version +{ + my($fio, $value) = @_; + + if ( defined($value) ) { + $fio->{protocol_version} = $value; + $fio->{digest}->protocol($fio->{protocol_version}); + } + return $fio->{protocol_version}; +} + +sub preserve_hard_links +{ + my($fio, $value) = @_; + + $fio->{preserve_hard_links} = $value if ( defined($value) ); + return $fio->{preserve_hard_links}; +} + sub logHandlerSet { my($fio, $sub) = @_; $fio->{logHandler} = $sub; + BackupPC::Xfer::RsyncDigest->logHandlerSet($sub); } # @@ -104,23 +141,70 @@ sub logHandlerSet # sub csumStart { - my($fio, $f, $needMD4, $defBlkSize) = @_; + my($fio, $f, $needMD4, $defBlkSize, $phase) = @_; $defBlkSize ||= $fio->{blockSize}; - my $attr = $fio->attribGet($f); + my $attr = $fio->attribGet($f, 1); $fio->{file} = $f; $fio->csumEnd if ( defined($fio->{csum}) ); return -1 if ( $attr->{type} != BPC_FTYPE_FILE ); + + # + # Rsync uses short checksums on the first phase. If the whole-file + # checksum fails, then the file is repeated with full checksums. + # So on phase 2 we verify the checksums if they are cached. + # + if ( ($phase > 0 || rand(1) < $fio->{cacheCheckProb}) + && $attr->{compress} + && $fio->{checksumSeed} == RSYNC_CSUMSEED_CACHE ) { + my($err, $d, $blkSize) = BackupPC::Xfer::RsyncDigest->digestStart( + $attr->{fullPath}, $attr->{size}, 0, + $defBlkSize, $fio->{checksumSeed}, + 0, $attr->{compress}, 0, + $fio->{protocol_version}); + if ( $err ) { + $fio->log("Can't get rsync digests from $attr->{fullPath}" + . " (err=$err, name=$f->{name})"); + $fio->{stats}{errorCnt}++; + return -1; + } + my($isCached, $isInvalid) = $d->isCached; + if ( $fio->{logLevel} >= 5 ) { + $fio->log("$attr->{fullPath} verify; cached = $isCached," + . " invalid = $isInvalid, phase = $phase"); + } + if ( $isCached || $isInvalid ) { + my $ret = BackupPC::Xfer::RsyncDigest->digestAdd( + $attr->{fullPath}, $blkSize, + $fio->{checksumSeed}, 1, # verify + $fio->{protocol_version} + ); + if ( $ret != 1 ) { + $fio->log("Bad cached digest for $attr->{fullPath} ($ret);" + . " fixed"); + $fio->{stats}{errorCnt}++; + } else { + $fio->log("$f->{name}: verified cached digest") + if ( $fio->{logLevel} >= 2 ); + } + } + $d->digestEnd; + } (my $err, $fio->{csum}, my $blkSize) = BackupPC::Xfer::RsyncDigest->digestStart($attr->{fullPath}, $attr->{size}, 0, $defBlkSize, $fio->{checksumSeed}, - $needMD4, $attr->{compress}, 1); + $needMD4, $attr->{compress}, 1, $fio->{protocol_version}); if ( $err ) { $fio->log("Can't get rsync digests from $attr->{fullPath}" . " (err=$err, name=$f->{name})"); $fio->{stats}{errorCnt}++; return -1; } + if ( $fio->{logLevel} >= 5 ) { + my($isCached, $invalid) = $fio->{csum}->isCached; + $fio->log("$attr->{fullPath} cache = $isCached," + . " invalid = $invalid, phase = $phase"); + } return $blkSize; } @@ -147,7 +231,7 @@ sub readStart { my($fio, $f) = @_; - my $attr = $fio->attribGet($f); + my $attr = $fio->attribGet($f, 1); $fio->{file} = $f; $fio->readEnd if ( defined($fio->{fh}) ); if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath}, @@ -192,9 +276,9 @@ sub checksumSeed $fio->{checksumSeed} = $checksumSeed; $fio->log("Checksum caching enabled (checksumSeed = $checksumSeed)") - if ( $fio->{logLevel} >= 1 && $checksumSeed == 32761 ); + if ( $fio->{logLevel} >= 1 && $checksumSeed == RSYNC_CSUMSEED_CACHE ); $fio->log("Checksum seed is $checksumSeed") - if ( $fio->{logLevel} >= 2 && $checksumSeed != 32761 ); + if ( $fio->{logLevel} >= 2 && $checksumSeed != RSYNC_CSUMSEED_CACHE ); } sub dirs @@ -245,14 +329,16 @@ sub viewCacheDir sub attribGetWhere { - my($fio, $f) = @_; - my($dir, $fname, $share, $shareM); + my($fio, $f, $noCache, $fname) = @_; + my($dir, $share, $shareM, $partial, $attr); - $fname = $f->{name}; - $fname = "$fio->{xfer}{pathHdrSrc}/$fname" + if ( !defined($fname) ) { + $fname = $f->{name}; + $fname = "$fio->{xfer}{pathHdrSrc}/$fname" if ( defined($fio->{xfer}{pathHdrSrc}) ); + } $fname =~ s{//+}{/}g; - if ( $fname =~ m{(.*)/(.*)} ) { + if ( $fname =~ m{(.*)/(.*)}s ) { $shareM = $fio->{shareM}; $dir = $1; $fname = $2; @@ -265,31 +351,72 @@ sub attribGetWhere $dir = ""; $fname = $fio->{share}; } - $fio->viewCacheDir($share, $dir); $shareM .= "/$dir" if ( $dir ne "" ); - if ( defined(my $attr = $fio->{viewCache}{$shareM}{$fname}) ) { - return ($attr, 0); - } elsif ( defined(my $attr = $fio->{partialCache}{$shareM}{$fname}) ) { - return ($attr, 1); + + if ( $noCache ) { + $share = $fio->{share} if ( !defined($share) ); + my $dirAttr = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir); + $attr = $dirAttr->{$fname}; } else { - return; + $fio->viewCacheDir($share, $dir); + if ( defined($attr = $fio->{viewCache}{$shareM}{$fname}) ) { + $partial = 0; + } elsif ( defined($attr = $fio->{partialCache}{$shareM}{$fname}) ) { + $partial = 1; + } else { + return; + } + if ( $attr->{mode} & S_HLINK_TARGET ) { + $attr->{hlink_self} = 1; + $attr->{mode} &= ~S_HLINK_TARGET; + } } + return ($attr, $partial); } sub attribGet { - my($fio, $f) = @_; + my($fio, $f, $doHardLink) = @_; my($attr) = $fio->attribGetWhere($f); + if ( $doHardLink && $attr->{type} == BPC_FTYPE_HARDLINK ) { + $fio->log("$attr->{fullPath}: opening for hardlink read" + . " (name = $f->{name})") if ( $fio->{logLevel} >= 4 ); + my $fh = BackupPC::FileZIO->open($attr->{fullPath}, 0, + $attr->{compress}); + my $target; + if ( defined($fh) ) { + $fh->read(\$target, 65536); + $fh->close; + $target =~ s/^\.?\/+//; + } else { + $fio->log("$attr->{fullPath}: can't open for hardlink read"); + $fio->{stats}{errorCnt}++; + $attr->{type} = BPC_FTYPE_FILE; + return $attr; + } + $target = "/$target" if ( $target !~ /^\// ); + $fio->log("$attr->{fullPath}: redirecting to $target") + if ( $fio->{logLevel} >= 4 ); + $target =~ s{^/+}{}; + ($attr) = $fio->attribGetWhere($f, 1, $target); + $fio->log(" ... now got $attr->{fullPath}") + if ( $fio->{logLevel} >= 4 ); + } return $attr; } sub mode2type { - my($fio, $mode) = @_; + my($fio, $f) = @_; + my $mode = $f->{mode}; if ( ($mode & S_IFMT) == S_IFREG ) { - return BPC_FTYPE_FILE; + if ( defined($f->{hlink}) && !$f->{hlink_self} ) { + return BPC_FTYPE_HARDLINK; + } else { + return BPC_FTYPE_FILE; + } } elsif ( ($mode & S_IFMT) == S_IFDIR ) { return BPC_FTYPE_DIR; } elsif ( ($mode & S_IFMT) == S_IFLNK ) { @@ -315,7 +442,9 @@ sub attribSet my($fio, $f, $placeHolder) = @_; my($dir, $file); - if ( $f->{name} =~ m{(.*)/(.*)} ) { + return if ( $placeHolder && $fio->{phase} > 0 ); + + if ( $f->{name} =~ m{(.*)/(.*)}s ) { $file = $2; $dir = "$fio->{shareM}/" . $1; } elsif ( $f->{name} eq "." ) { @@ -326,10 +455,13 @@ sub attribSet $file = $f->{name}; } - if ( !defined($fio->{attribLastDir}) || $fio->{attribLastDir} ne $dir ) { + if ( $dir ne "" + && (!defined($fio->{attribLastDir}) || $fio->{attribLastDir} ne $dir) ) { # # Flush any directories that don't match the first part - # of the new directory + # of the new directory. Don't flush the top-level directory + # (ie: $dir eq "") since the "." might get sorted in the middle + # of other top-level directories or files. # foreach my $d ( keys(%{$fio->{attrib}}) ) { next if ( $d eq "" || "$dir/" =~ m{^\Q$d/} ); @@ -338,21 +470,36 @@ sub attribSet $fio->{attribLastDir} = $dir; } if ( !exists($fio->{attrib}{$dir}) ) { + $fio->log("attribSet: dir=$dir not found") if ( $fio->{logLevel} >= 4 ); $fio->{attrib}{$dir} = BackupPC::Attrib->new({ compress => $fio->{xfer}{compress}, }); - my $path = $fio->{outDir} . $dir; - if ( -f $fio->{attrib}{$dir}->fileName($path) - && !$fio->{attrib}{$dir}->read($path) ) { - $fio->log(sprintf("Unable to read attribute file %s", + my $dirM = $dir; + $dirM = $1 . "/" . $fio->{bpc}->fileNameMangle($2) + if ( $dirM =~ m{(.*?)/(.*)}s ); + my $path = $fio->{outDir} . $dirM; + if ( -f $fio->{attrib}{$dir}->fileName($path) ) { + if ( !$fio->{attrib}{$dir}->read($path) ) { + $fio->log(sprintf("Unable to read attribute file %s", $fio->{attrib}{$dir}->fileName($path))); + } else { + $fio->log(sprintf("attribRead file %s", + $fio->{attrib}{$dir}->fileName($path))) + if ( $fio->{logLevel} >= 4 ); + } } + } else { + $fio->log("attribSet: dir=$dir exists") if ( $fio->{logLevel} >= 4 ); } - $fio->log("attribSet(dir=$dir, file=$file)") if ( $fio->{logLevel} >= 4 ); + $fio->log("attribSet(dir=$dir, file=$file, size=$f->{size}, placeholder=$placeHolder)") + if ( $fio->{logLevel} >= 4 ); + my $mode = $f->{mode}; + + $mode |= S_HLINK_TARGET if ( $f->{hlink_self} ); $fio->{attrib}{$dir}->set($file, { - type => $fio->mode2type($f->{mode}), - mode => $f->{mode}, + type => $fio->mode2type($f), + mode => $mode, uid => $f->{uid}, gid => $f->{gid}, size => $placeHolder ? -1 : $f->{size}, @@ -376,6 +523,7 @@ sub attribWrite return; } return if ( !defined($fio->{attrib}{$d}) ); + # # Set deleted files in the attributes. Any file in the view # that doesn't have attributes is flagged as deleted for @@ -387,7 +535,7 @@ sub attribWrite my $dir; my $share; - $dir = $1 if ( $d =~ m{.+?/(.*)} ); + $dir = $1 if ( $d =~ m{.+?/(.*)}s ); $fio->viewCacheDir(undef, $dir); ##print("attribWrite $d,$dir\n"); ##$Data::Dumper::Indent = 1; @@ -399,7 +547,7 @@ sub attribWrite if ( defined($fio->{viewCache}{$d}) ) { foreach my $f ( keys(%{$fio->{viewCache}{$d}}) ) { my $name = $f; - $name = "$1/$name" if ( $d =~ m{.*?/(.*)} ); + $name = "$1/$name" if ( $d =~ m{.*?/(.*)}s ); if ( defined(my $a = $fio->{attrib}{$d}->get($f)) ) { # # delete temporary attributes (skipped files) @@ -409,9 +557,10 @@ sub attribWrite $fio->logFileAction("skip", { %{$fio->{viewCache}{$d}{$f}}, name => $name, - }) if ( $fio->{logLevel} >= 2 ); + }) if ( $fio->{logLevel} >= 2 + && $a->{type} == BPC_FTYPE_FILE ); } - } elsif ( !$fio->{full} ) { + } elsif ( $fio->{phase} == 0 && !$fio->{full} ) { ##print("Delete file $f\n"); $fio->logFileAction("delete", { %{$fio->{viewCache}{$d}{$f}}, @@ -429,12 +578,12 @@ sub attribWrite } } } - if ( $fio->{attrib}{$d}->fileCount ) { + if ( $fio->{attrib}{$d}->fileCount || $fio->{phase} > 0 ) { my $data = $fio->{attrib}{$d}->writeData; my $dirM = $d; $dirM = $1 . "/" . $fio->{bpc}->fileNameMangle($2) - if ( $dirM =~ m{(.*?)/(.*)} ); + if ( $dirM =~ m{(.*?)/(.*)}s ); my $fileName = $fio->{attrib}{$d}->fileName("$fio->{outDir}$dirM"); $fio->log("attribWrite(dir=$d) -> $fileName") if ( $fio->{logLevel} >= 4 ); @@ -453,7 +602,10 @@ sub processClose my($exists, $digest, $outSize, $errs) = $poolWrite->close; $fileName =~ s{^/+}{}; - $fio->log(@$errs) if ( defined($errs) && @$errs ); + if ( defined($errs) && @$errs ) { + $fio->log(@$errs); + $fio->{stats}{errorCnt} += @$errs; + } if ( $doStats ) { $fio->{stats}{TotalFileCnt}++; $fio->{stats}{TotalFileSize} += $origSize; @@ -484,7 +636,7 @@ sub statsGet sub makePath { my($fio, $f) = @_; - my $name = $1 if ( $f->{name} =~ /(.*)/ ); + my $name = $1 if ( $f->{name} =~ /(.*)/s ); my $path; if ( $name eq "." ) { @@ -494,8 +646,8 @@ sub makePath } $fio->logFileAction("create", $f) if ( $fio->{logLevel} >= 1 ); $fio->log("makePath($path, 0777)") if ( $fio->{logLevel} >= 5 ); - $path = $1 if ( $path =~ /(.*)/ ); - File::Path::mkpath($path, 0, 0777) if ( !-d $path ); + $path = $1 if ( $path =~ /(.*)/s ); + eval { File::Path::mkpath($path, 0, 0777) } if ( !-d $path ); return $fio->attribSet($f) if ( -d $path ); $fio->log("Can't create directory $path"); $fio->{stats}{errorCnt}++; @@ -508,23 +660,38 @@ sub makePath sub makeSpecial { my($fio, $f) = @_; - my $name = $1 if ( $f->{name} =~ /(.*)/ ); + my $name = $1 if ( $f->{name} =~ /(.*)/s ); my $fNameM = $fio->{bpc}->fileNameMangle($name); my $path = $fio->{outDirSh} . $fNameM; my $attr = $fio->attribGet($f); my $str = ""; - my $type = $fio->mode2type($f->{mode}); + my $type = $fio->mode2type($f); $fio->log("makeSpecial($path, $type, $f->{mode})") if ( $fio->{logLevel} >= 5 ); if ( $type == BPC_FTYPE_CHARDEV || $type == BPC_FTYPE_BLOCKDEV ) { my($major, $minor, $fh, $fileData); - $major = $f->{rdev} >> 8; - $minor = $f->{rdev} & 0xff; + if ( defined($f->{rdev_major}) ) { + $major = $f->{rdev_major}; + $minor = $f->{rdev_minor}; + } else { + $major = $f->{rdev} >> 8; + $minor = $f->{rdev} & 0xff; + } $str = "$major,$minor"; } elsif ( ($f->{mode} & S_IFMT) == S_IFLNK ) { $str = $f->{link}; + } elsif ( ($f->{mode} & S_IFMT) == S_IFREG ) { + # + # this is a hardlink + # + if ( !defined($f->{hlink}) ) { + $fio->log("Error: makeSpecial($path, $type, $f->{mode}) called" + . " on a regular non-hardlink file"); + return 1; + } + $str = $f->{hlink}; } # # Now see if the file is different, or this is a full, in which @@ -533,12 +700,13 @@ sub makeSpecial my($fh, $fileData); if ( $fio->{full} || !defined($attr) - || $attr->{type} != $fio->mode2type($f->{mode}) - || $attr->{mtime} != $f->{mtime} - || $attr->{size} != $f->{size} - || $attr->{uid} != $f->{uid} - || $attr->{gid} != $f->{gid} - || $attr->{mode} != $f->{mode} + || $attr->{type} != $type + || $attr->{mtime} != $f->{mtime} + || $attr->{size} != $f->{size} + || $attr->{uid} != $f->{uid} + || $attr->{gid} != $f->{gid} + || $attr->{mode} != $f->{mode} + || $attr->{hlink_self} != $f->{hlink_self} || !defined($fh = BackupPC::FileZIO->open($attr->{fullPath}, 0, $attr->{compress})) || $fh->read(\$fileData, length($str) + 1) != length($str) @@ -558,6 +726,22 @@ sub makeSpecial $fh->close if ( defined($fh) ); } +# +# Make a hardlink. Returns non-zero on error. +# This actually gets called twice for each hardlink. +# Once as the file list is processed, and again at +# the end. BackupPC does them as it goes (since it is +# just saving the hardlink info and not actually making +# hardlinks). +# +sub makeHardLink +{ + my($fio, $f, $end) = @_; + + return if ( $end ); + return $fio->makeSpecial($f) if ( !$f->{hlink_self} ); +} + sub unlink { my($fio, $path) = @_; @@ -597,6 +781,15 @@ sub logFileAction my $owner = "$f->{uid}/$f->{gid}"; my $type = (("", "p", "c", "", "d", "", "b", "", "", "", "l", "", "s")) [($f->{mode} & S_IFMT) >> 12]; + my $name = $f->{name}; + + if ( ($f->{mode} & S_IFMT) == S_IFLNK ) { + $name .= " -> $f->{link}"; + } elsif ( ($f->{mode} & S_IFMT) == S_IFREG + && defined($f->{hlink}) && !$f->{hlink_self} ) { + $name .= " -> $f->{hlink}"; + } + $name =~ s/\n/\\n/g; $fio->log(sprintf(" %-6s %1s%4o %9s %11.0f %s", $action, @@ -604,7 +797,7 @@ sub logFileAction $f->{mode} & 07777, $owner, $f->{size}, - $f->{name})); + $name)); } # @@ -672,6 +865,32 @@ sub fileDeltaRxStart . " ($fio->{rxFile}{size} vs $rxSize)") if ( $fio->{logLevel} >= 5 ); } + # + # If compression was off and now on, or on and now off, then + # don't do an exact match. + # + if ( defined($fio->{rxLocalAttr}) + && !$fio->{rxLocalAttr}{compress} != !$fio->{xfer}{compress} ) { + $fio->{rxMatchBlk} = undef; # compression changed, so no file match + $fio->log("$fio->{rxFile}{name}: compression changed, so no match" + . " ($fio->{rxLocalAttr}{compress} vs $fio->{xfer}{compress})") + if ( $fio->{logLevel} >= 4 ); + } + # + # If the local file is a hardlink then no match + # + if ( defined($fio->{rxLocalAttr}) + && $fio->{rxLocalAttr}{type} == BPC_FTYPE_HARDLINK ) { + $fio->{rxMatchBlk} = undef; + $fio->log("$fio->{rxFile}{name}: no match on hardlinks") + if ( $fio->{logLevel} >= 4 ); + my $fCopy; + # need to copy since hardlink attribGet overwrites the name + %{$fCopy} = %$f; + $fio->{rxHLinkAttr} = $fio->attribGet($fCopy, 1); # hardlink attributes + } else { + delete($fio->{rxHLinkAttr}); + } delete($fio->{rxInFd}); delete($fio->{rxOutFd}); delete($fio->{rxDigest}); @@ -714,7 +933,7 @@ sub fileDeltaRxNext # need to open an output file where we will build the # new version. # - $fio->{rxFile}{name} =~ /(.*)/; + $fio->{rxFile}{name} =~ /(.*)/s; my $rxOutFileRel = "$fio->{shareM}/" . $fio->{bpc}->fileNameMangle($1); my $rxOutFile = $fio->{outDir} . $rxOutFileRel; $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc}, @@ -724,7 +943,8 @@ sub fileDeltaRxNext if ( $fio->{logLevel} >= 9 ); $fio->{rxOutFile} = $rxOutFile; $fio->{rxOutFileRel} = $rxOutFileRel; - $fio->{rxDigest} = File::RsyncP::Digest->new; + $fio->{rxDigest} = File::RsyncP::Digest->new(); + $fio->{rxDigest}->protocol($fio->{protocol_version}); $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed})); } if ( defined($fio->{rxMatchBlk}) @@ -740,12 +960,15 @@ sub fileDeltaRxNext my $attr = $fio->{rxLocalAttr}; my $fh; if ( !defined($fio->{rxInFd}) && !defined($fio->{rxInData}) ) { + my $inPath = $attr->{fullPath}; + $inPath = $fio->{rxHLinkAttr}{fullPath} + if ( defined($fio->{rxHLinkAttr}) ); if ( $attr->{compress} ) { if ( !defined($fh = BackupPC::FileZIO->open( - $attr->{fullPath}, + $inPath, 0, $attr->{compress})) ) { - $fio->log("Can't open $attr->{fullPath}"); + $fio->log("Can't open $inPath"); $fio->{stats}{errorCnt}++; return -1; } @@ -796,12 +1019,12 @@ sub fileDeltaRxNext } $fh->close; } else { - if ( open(F, "<", $attr->{fullPath}) ) { + if ( open(F, "<", $inPath) ) { binmode(F); $fio->{rxInFd} = *F; $fio->{rxInName} = $attr->{fullPath}; } else { - $fio->log("Unable to open $attr->{fullPath}"); + $fio->log("Unable to open $inPath"); $fio->{stats}{errorCnt}++; return -1; } @@ -814,7 +1037,7 @@ sub fileDeltaRxNext my $seekPosn = $fio->{rxMatchBlk} * $fio->{rxBlkSize}; if ( defined($fio->{rxInFd}) && !sysseek($fio->{rxInFd}, $seekPosn, 0) ) { - $fio->log("Unable to seek $attr->{rxInName} to $seekPosn"); + $fio->log("Unable to seek $fio->{rxInName} to $seekPosn"); $fio->{stats}{errorCnt}++; return -1; } @@ -876,12 +1099,13 @@ sub fileDeltaRxNext # sub fileDeltaRxDone { - my($fio, $md4) = @_; - my $name = $1 if ( $fio->{rxFile}{name} =~ /(.*)/ ); + my($fio, $md4, $phase) = @_; + my $name = $1 if ( $fio->{rxFile}{name} =~ /(.*)/s ); my $ret; close($fio->{rxInFd}) if ( defined($fio->{rxInFd}) ); unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" ); + $fio->{phase} = $phase; # # Check the final md4 digest @@ -903,12 +1127,18 @@ sub fileDeltaRxDone = BackupPC::Xfer::RsyncDigest->digestStart( $attr->{fullPath}, $attr->{size}, 0, 2048, $fio->{checksumSeed}, 1, - $attr->{compress}); + $attr->{compress}, 1, + $fio->{protocol_version}); if ( $err ) { $fio->log("Can't open $attr->{fullPath} for MD4" . " check (err=$err, $name)"); $fio->{stats}{errorCnt}++; } else { + if ( $fio->{logLevel} >= 5 ) { + my($isCached, $invalid) = $csum->isCached; + $fio->log("MD4 $attr->{fullPath} cache = $isCached," + . " invalid = $invalid"); + } $newDigest = $csum->digestEnd; } $fio->{rxSize} = $attr->{size}; @@ -916,7 +1146,8 @@ sub fileDeltaRxDone # # Empty file; just create an empty file digest # - $fio->{rxDigest} = File::RsyncP::Digest->new; + $fio->{rxDigest} = File::RsyncP::Digest->new(); + $fio->{rxDigest}->protocol($fio->{protocol_version}); $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed})); $newDigest = $fio->{rxDigest}->digest; } @@ -930,8 +1161,14 @@ sub fileDeltaRxDone $fio->log("$name got digests $md4Str vs $newStr") } if ( $md4 ne $newDigest ) { - $fio->log("$name: fatal error: md4 doesn't match"); - $fio->{stats}{errorCnt}++; + if ( $phase > 0 ) { + $fio->log("$name: fatal error: md4 doesn't match on retry;" + . " file removed"); + $fio->{stats}{errorCnt}++; + } else { + $fio->log("$name: md4 doesn't match: will retry in phase 1;" + . " file removed"); + } if ( defined($fio->{rxOutFd}) ) { $fio->{rxOutFd}->close; unlink($fio->{rxOutFile}); @@ -964,11 +1201,13 @@ sub fileDeltaRxDone my $f = $fio->{rxFile}; $fio->logFileAction("same", $f) if ( $fio->{logLevel} >= 1 ); if ( $fio->{full} - || $attr->{type} != $f->{type} - || $attr->{mtime} != $f->{mtime} - || $attr->{size} != $f->{size} - || $attr->{gid} != $f->{gid} - || $attr->{mode} != $f->{mode} ) { + || $attr->{type} != $f->{type} + || $attr->{mtime} != $f->{mtime} + || $attr->{size} != $f->{size} + || $attr->{uid} != $f->{uid} + || $attr->{gid} != $f->{gid} + || $attr->{mode} != $f->{mode} + || $attr->{hlink_self} != $f->{hlink_self} ) { # # In the full case, or if the attributes are different, # we need to make a link from the previous file and @@ -976,22 +1215,38 @@ sub fileDeltaRxDone # my $rxOutFile = $fio->{outDirSh} . $fio->{bpc}->fileNameMangle($name); - if ( !link($attr->{fullPath}, $rxOutFile) ) { - $fio->log("Unable to link $attr->{fullPath} to $rxOutFile"); - $fio->{stats}{errorCnt}++; - $ret = -1; - } else { - # - # Cumulate the stats - # - $fio->{stats}{TotalFileCnt}++; - $fio->{stats}{TotalFileSize} += $fio->{rxSize}; - $fio->{stats}{ExistFileCnt}++; - $fio->{stats}{ExistFileSize} += $fio->{rxSize}; - $fio->{stats}{ExistFileCompSize} += -s $rxOutFile; - $fio->{rxFile}{size} = $fio->{rxSize}; - $ret = $fio->attribSet($fio->{rxFile}); - } + my($exists, $digest, $origSize, $outSize, $errs) + = BackupPC::PoolWrite::LinkOrCopy( + $fio->{bpc}, + $attr->{fullPath}, + $attr->{compress}, + $rxOutFile, + $fio->{xfer}{compress}); + # + # Cumulate the stats + # + $fio->{stats}{TotalFileCnt}++; + $fio->{stats}{TotalFileSize} += $fio->{rxSize}; + $fio->{stats}{ExistFileCnt}++; + $fio->{stats}{ExistFileSize} += $fio->{rxSize}; + $fio->{stats}{ExistFileCompSize} += -s $rxOutFile; + $fio->{rxFile}{size} = $fio->{rxSize}; + $ret = $fio->attribSet($fio->{rxFile}); + $fio->log(@$errs) if ( defined($errs) && @$errs ); + + if ( !$exists && $outSize > 0 ) { + # + # the hard link failed, most likely because the target + # file has too many links. We have copied the file + # instead, so add this to the new file list. + # + my $rxOutFileRel = "$fio->{shareM}/" + . $fio->{bpc}->fileNameMangle($name); + $rxOutFileRel =~ s{^/+}{}; + my $fh = $fio->{newFilesFH}; + print($fh "$digest $origSize $rxOutFileRel\n") + if ( defined($fh) ); + } } } else { my $exist = $fio->processClose($fio->{rxOutFd}, @@ -1018,11 +1273,15 @@ sub fileListEltSend my($a, $fio, $fList, $outputFunc) = @_; my $name = $a->{relPath}; my $n = $name; - my $type = $fio->mode2type($a->{mode}); + my $type = $a->{type}; my $extraAttribs = {}; + if ( $a->{mode} & S_HLINK_TARGET ) { + $a->{hlink_self} = 1; + $a->{mode} &= ~S_HLINK_TARGET; + } $n =~ s/^\Q$fio->{xfer}{pathHdrSrc}//; - $fio->log("Sending $name (remote=$n)") if ( $fio->{logLevel} >= 4 ); + $fio->log("Sending $name (remote=$n) type = $type") if ( $fio->{logLevel} >= 1 ); if ( $type == BPC_FTYPE_CHARDEV || $type == BPC_FTYPE_BLOCKDEV || $type == BPC_FTYPE_SYMLINK ) { @@ -1047,7 +1306,11 @@ sub fileListEltSend # Note: char/block devices have $a->{size} = 0, so we # can't do an error check on $rdSize. # - $extraAttribs = { rdev => $1 * 256 + $2 }; + $extraAttribs = { + rdev => $1 * 256 + $2, + rdev_major => $1, + rdev_minor => $2, + }; } else { $fio->log("$name: unexpected special file contents $str"); $fio->{stats}{errorCnt}++; @@ -1058,26 +1321,75 @@ sub fileListEltSend $fio->log("$name: can't open"); $fio->{stats}{errorCnt}++; } + } elsif ( $fio->{preserve_hard_links} + && ($type == BPC_FTYPE_HARDLINK || $type == BPC_FTYPE_FILE) + && ($type == BPC_FTYPE_HARDLINK + || $fio->{protocol_version} < 27 + || $a->{hlink_self}) ) { + # + # Fill in fake inode information so that the remote rsync + # can correctly create hardlinks. + # + $name =~ s/^\.?\/+//; + my($target, $inode); + + if ( $type == BPC_FTYPE_HARDLINK ) { + my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0, + $a->{compress}); + if ( defined($fh) ) { + $fh->read(\$target, 65536); + $fh->close; + $target =~ s/^\.?\/+//; + if ( defined($fio->{hlinkFile2Num}{$target}) ) { + $inode = $fio->{hlinkFile2Num}{$target}; + } else { + $inode = $fio->{fileListCnt}; + $fio->{hlinkFile2Num}{$target} = $inode; + } + } else { + $fio->log("$a->{fullPath}: can't open for hardlink"); + $fio->{stats}{errorCnt}++; + } + } elsif ( $a->{hlink_self} ) { + if ( defined($fio->{hlinkFile2Num}{$name}) ) { + $inode = $fio->{hlinkFile2Num}{$name}; + } else { + $inode = $fio->{fileListCnt}; + $fio->{hlinkFile2Num}{$name} = $inode; + } + } + $inode = $fio->{fileListCnt} if ( !defined($inode) ); + $fio->log("$name: setting inode to $inode"); + $extraAttribs = { + %$extraAttribs, + dev => 0, + inode => $inode, + }; } my $f = { - name => $n, - #dev => 0, # later, when we support hardlinks - #inode => 0, # later, when we support hardlinks - mode => $a->{mode}, - uid => $a->{uid}, - gid => $a->{gid}, - mtime => $a->{mtime}, - size => $a->{size}, - %$extraAttribs, + name => $n, + mode => $a->{mode} & ~S_HLINK_TARGET, + uid => $a->{uid}, + gid => $a->{gid}, + mtime => $a->{mtime}, + size => $a->{size}, + %$extraAttribs, }; + my $logName = $f->{name}; + from_to($f->{name}, "utf8", $fio->{clientCharset}) + if ( $fio->{clientCharset} ne "" ); $fList->encode($f); - $f->{name} = "$fio->{xfer}{pathHdrDest}/$f->{name}"; - $f->{name} =~ s{//+}{/}g; + + $logName = "$fio->{xfer}{pathHdrDest}/$logName"; + $logName =~ s{//+}{/}g; + $f->{name} = $logName; $fio->logFileAction("restore", $f) if ( $fio->{logLevel} >= 1 ); + &$outputFunc($fList->encodeData); # # Cumulate stats # + $fio->{fileListCnt}++; if ( $type != BPC_FTYPE_DIR ) { $fio->{stats}{TotalFileCnt}++; $fio->{stats}{TotalFileSize} += $a->{size}; @@ -1094,6 +1406,8 @@ sub fileListSend # $fio->log("fileListSend: sending file list: " . join(" ", @{$fio->{fileList}})) if ( $fio->{logLevel} >= 4 ); + $fio->{fileListCnt} = 0; + $fio->{hlinkFile2Num} = {}; foreach my $name ( @{$fio->{fileList}} ) { $fio->{view}->find($fio->{xfer}{bkupSrcNum}, $fio->{xfer}{bkupSrcShare},