1 #============================================================= -*-perl-*-
8 # Craig Barratt <cbarratt@users.sourceforge.net>
11 # Copyright (C) 2002-2003 Craig Barratt
13 #========================================================================
15 # Version 2.1.0, released 20 Jun 2004.
17 # See http://backuppc.sourceforge.net.
19 #========================================================================
21 package BackupPC::Xfer::RsyncFileIO;
25 use BackupPC::Attrib qw(:all);
27 use BackupPC::Xfer::RsyncDigest qw(:all);
28 use BackupPC::PoolWrite;
30 use constant S_IFMT => 0170000; # type of file
31 use constant S_IFDIR => 0040000; # directory
32 use constant S_IFCHR => 0020000; # character special
33 use constant S_IFBLK => 0060000; # block special
34 use constant S_IFREG => 0100000; # regular
35 use constant S_IFLNK => 0120000; # symbolic link
36 use constant S_IFSOCK => 0140000; # socket
37 use constant S_IFIFO => 0010000; # fifo
39 use vars qw( $RsyncLibOK );
42 eval "use File::RsyncP::Digest";
45 # Rsync module doesn't exist.
55 my($class, $options) = @_;
57 return if ( !$RsyncLibOK );
62 digest => File::RsyncP::Digest->new,
65 logHandler => \&logHandler,
72 ExistFileCompSize => 0,
77 $fio->{shareM} = $fio->{bpc}->fileNameEltMangle($fio->{share});
78 $fio->{outDir} = "$fio->{xfer}{outDir}/new/";
79 $fio->{outDirSh} = "$fio->{outDir}/$fio->{shareM}/";
80 $fio->{view} = BackupPC::View->new($fio->{bpc}, $fio->{client},
82 $fio->{full} = $fio->{xfer}{type} eq "full" ? 1 : 0;
83 $fio->{newFilesFH} = $fio->{xfer}{newFilesFH};
84 $fio->{partialNum} = undef if ( !$fio->{full} );
90 my($fio, $value) = @_;
92 $fio->{blockSize} = $value if ( defined($value) );
93 return $fio->{blockSize};
99 $fio->{logHandler} = $sub;
100 BackupPC::Xfer::RsyncDigest->logHandlerSet($sub);
104 # Setup rsync checksum computation for the given file.
108 my($fio, $f, $needMD4, $defBlkSize, $phase) = @_;
110 $defBlkSize ||= $fio->{blockSize};
111 my $attr = $fio->attribGet($f);
113 $fio->csumEnd if ( defined($fio->{csum}) );
114 return -1 if ( $attr->{type} != BPC_FTYPE_FILE );
116 # Rsync uses short checksums on the first phase. If the whole-file
117 # checksum fails, then the file is repeated with full checksums.
118 # So on phase 2 we verify the checksums if they are cached.
120 if ( ($phase > 0 || rand(1) < $fio->{cacheCheckProb})
122 && $fio->{checksumSeed} == RSYNC_CSUMSEED_CACHE ) {
123 my($err, $d, $blkSize) = BackupPC::Xfer::RsyncDigest->digestStart(
124 $attr->{fullPath}, $attr->{size}, 0,
125 $defBlkSize, $fio->{checksumSeed},
126 0, $attr->{compress}, 0);
127 my($isCached, $isInvalid) = $d->isCached;
128 if ( $fio->{logLevel} >= 5 ) {
129 $fio->log("$attr->{fullPath} verify; cached = $isCached,"
130 . " invalid = $isInvalid, phase = $phase");
132 if ( $isCached || $isInvalid ) {
133 my $ret = BackupPC::Xfer::RsyncDigest->digestAdd(
134 $attr->{fullPath}, $blkSize,
135 $fio->{checksumSeed}, 1 # verify
138 $fio->log("Bad cached digest for $attr->{fullPath} ($ret);"
140 $fio->{stats}{errorCnt}++;
142 $fio->log("$f->{name}: verified cached digest")
143 if ( $fio->{logLevel} >= 2 );
148 (my $err, $fio->{csum}, my $blkSize)
149 = BackupPC::Xfer::RsyncDigest->digestStart($attr->{fullPath},
150 $attr->{size}, 0, $defBlkSize, $fio->{checksumSeed},
151 $needMD4, $attr->{compress}, 1);
152 if ( $fio->{logLevel} >= 5 ) {
153 my($isCached, $invalid) = $fio->{csum}->isCached;
154 $fio->log("$attr->{fullPath} cache = $isCached,"
155 . " invalid = $invalid, phase = $phase");
158 $fio->log("Can't get rsync digests from $attr->{fullPath}"
159 . " (err=$err, name=$f->{name})");
160 $fio->{stats}{errorCnt}++;
168 my($fio, $num, $csumLen, $blockSize) = @_;
173 return if ( !defined($fio->{csum}) );
174 return $fio->{csum}->digestGet($num, $csumLen);
181 return if ( !defined($fio->{csum}) );
182 return $fio->{csum}->digestEnd();
189 my $attr = $fio->attribGet($f);
191 $fio->readEnd if ( defined($fio->{fh}) );
192 if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath},
194 $attr->{compress})) ) {
195 $fio->log("Can't open $attr->{fullPath} (name=$f->{name})");
196 $fio->{stats}{errorCnt}++;
199 $fio->log("$f->{name}: opened for read") if ( $fio->{logLevel} >= 4 );
208 return if ( !defined($fio->{fh}) );
209 if ( $fio->{fh}->read(\$fileData, $num) <= 0 ) {
210 return $fio->readEnd;
212 $fio->log(sprintf("read returns %d bytes", length($fileData)))
213 if ( $fio->{logLevel} >= 8 );
221 return if ( !defined($fio->{fh}) );
223 $fio->log("closing $fio->{file}{name})") if ( $fio->{logLevel} >= 8 );
230 my($fio, $checksumSeed) = @_;
232 $fio->{checksumSeed} = $checksumSeed;
233 $fio->log("Checksum caching enabled (checksumSeed = $checksumSeed)")
234 if ( $fio->{logLevel} >= 1 && $checksumSeed == RSYNC_CSUMSEED_CACHE );
235 $fio->log("Checksum seed is $checksumSeed")
236 if ( $fio->{logLevel} >= 2 && $checksumSeed != RSYNC_CSUMSEED_CACHE );
241 my($fio, $localDir, $remoteDir) = @_;
243 $fio->{localDir} = $localDir;
244 $fio->{remoteDir} = $remoteDir;
249 my($fio, $share, $dir) = @_;
252 #$fio->log("viewCacheDir($share, $dir)");
253 if ( !defined($share) ) {
254 $share = $fio->{share};
255 $shareM = $fio->{shareM};
257 $shareM = $fio->{bpc}->fileNameEltMangle($share);
259 $shareM = "$shareM/$dir" if ( $dir ne "" );
260 return if ( defined($fio->{viewCache}{$shareM}) );
262 # purge old cache entries (ie: those that don't match the
263 # first part of $dir).
265 foreach my $d ( keys(%{$fio->{viewCache}}) ) {
266 delete($fio->{viewCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
269 # fetch new directory attributes
271 $fio->{viewCache}{$shareM}
272 = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir);
274 # also cache partial backup attrib data too
276 if ( defined($fio->{partialNum}) ) {
277 foreach my $d ( keys(%{$fio->{partialCache}}) ) {
278 delete($fio->{partialCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
280 $fio->{partialCache}{$shareM}
281 = $fio->{view}->dirAttrib($fio->{partialNum}, $share, $dir);
288 my($dir, $fname, $share, $shareM);
291 $fname = "$fio->{xfer}{pathHdrSrc}/$fname"
292 if ( defined($fio->{xfer}{pathHdrSrc}) );
293 $fname =~ s{//+}{/}g;
294 if ( $fname =~ m{(.*)/(.*)} ) {
295 $shareM = $fio->{shareM};
298 } elsif ( $fname ne "." ) {
299 $shareM = $fio->{shareM};
305 $fname = $fio->{share};
307 $fio->viewCacheDir($share, $dir);
308 $shareM .= "/$dir" if ( $dir ne "" );
309 if ( defined(my $attr = $fio->{viewCache}{$shareM}{$fname}) ) {
311 } elsif ( defined(my $attr = $fio->{partialCache}{$shareM}{$fname}) ) {
322 my($attr) = $fio->attribGetWhere($f);
328 my($fio, $mode) = @_;
330 if ( ($mode & S_IFMT) == S_IFREG ) {
331 return BPC_FTYPE_FILE;
332 } elsif ( ($mode & S_IFMT) == S_IFDIR ) {
333 return BPC_FTYPE_DIR;
334 } elsif ( ($mode & S_IFMT) == S_IFLNK ) {
335 return BPC_FTYPE_SYMLINK;
336 } elsif ( ($mode & S_IFMT) == S_IFCHR ) {
337 return BPC_FTYPE_CHARDEV;
338 } elsif ( ($mode & S_IFMT) == S_IFBLK ) {
339 return BPC_FTYPE_BLOCKDEV;
340 } elsif ( ($mode & S_IFMT) == S_IFIFO ) {
341 return BPC_FTYPE_FIFO;
342 } elsif ( ($mode & S_IFMT) == S_IFSOCK ) {
343 return BPC_FTYPE_SOCKET;
345 return BPC_FTYPE_UNKNOWN;
350 # Set the attributes for a file. Returns non-zero on error.
354 my($fio, $f, $placeHolder) = @_;
357 if ( $f->{name} =~ m{(.*)/(.*)} ) {
359 $dir = "$fio->{shareM}/" . $1;
360 } elsif ( $f->{name} eq "." ) {
362 $file = $fio->{share};
364 $dir = $fio->{shareM};
368 if ( !defined($fio->{attribLastDir}) || $fio->{attribLastDir} ne $dir ) {
370 # Flush any directories that don't match the first part
371 # of the new directory
373 foreach my $d ( keys(%{$fio->{attrib}}) ) {
374 next if ( $d eq "" || "$dir/" =~ m{^\Q$d/} );
375 $fio->attribWrite($d);
377 $fio->{attribLastDir} = $dir;
379 if ( !exists($fio->{attrib}{$dir}) ) {
380 $fio->{attrib}{$dir} = BackupPC::Attrib->new({
381 compress => $fio->{xfer}{compress},
383 my $path = $fio->{outDir} . $dir;
384 if ( -f $fio->{attrib}{$dir}->fileName($path)
385 && !$fio->{attrib}{$dir}->read($path) ) {
386 $fio->log(sprintf("Unable to read attribute file %s",
387 $fio->{attrib}{$dir}->fileName($path)));
390 $fio->log("attribSet(dir=$dir, file=$file)") if ( $fio->{logLevel} >= 4 );
392 $fio->{attrib}{$dir}->set($file, {
393 type => $fio->mode2type($f->{mode}),
397 size => $placeHolder ? -1 : $f->{size},
398 mtime => $f->{mtime},
408 if ( !defined($d) ) {
410 # flush all entries (in reverse order)
412 foreach $d ( sort({$b cmp $a} keys(%{$fio->{attrib}})) ) {
413 $fio->attribWrite($d);
417 return if ( !defined($fio->{attrib}{$d}) );
419 # Set deleted files in the attributes. Any file in the view
420 # that doesn't have attributes is flagged as deleted for
421 # incremental dumps. All files sent by rsync have attributes
422 # temporarily set so we can do deletion detection. We also
423 # prune these temporary attributes.
429 $dir = $1 if ( $d =~ m{.+?/(.*)} );
430 $fio->viewCacheDir(undef, $dir);
431 ##print("attribWrite $d,$dir\n");
432 ##$Data::Dumper::Indent = 1;
433 ##$fio->log("attribWrite $d,$dir");
434 ##$fio->log("viewCacheLogKeys = ", keys(%{$fio->{viewCache}}));
435 ##$fio->log("attribKeys = ", keys(%{$fio->{attrib}}));
436 ##print "viewCache = ", Dumper($fio->{attrib});
437 ##print "attrib = ", Dumper($fio->{attrib});
438 if ( defined($fio->{viewCache}{$d}) ) {
439 foreach my $f ( keys(%{$fio->{viewCache}{$d}}) ) {
441 $name = "$1/$name" if ( $d =~ m{.*?/(.*)} );
442 if ( defined(my $a = $fio->{attrib}{$d}->get($f)) ) {
444 # delete temporary attributes (skipped files)
446 if ( $a->{size} < 0 ) {
447 $fio->{attrib}{$d}->set($f, undef);
448 $fio->logFileAction("skip", {
449 %{$fio->{viewCache}{$d}{$f}},
451 }) if ( $fio->{logLevel} >= 2 );
453 } elsif ( !$fio->{full} ) {
454 ##print("Delete file $f\n");
455 $fio->logFileAction("delete", {
456 %{$fio->{viewCache}{$d}{$f}},
458 }) if ( $fio->{logLevel} >= 1 );
459 $fio->{attrib}{$d}->set($f, {
460 type => BPC_FTYPE_DELETED,
471 if ( $fio->{attrib}{$d}->fileCount ) {
472 my $data = $fio->{attrib}{$d}->writeData;
475 $dirM = $1 . "/" . $fio->{bpc}->fileNameMangle($2)
476 if ( $dirM =~ m{(.*?)/(.*)} );
477 my $fileName = $fio->{attrib}{$d}->fileName("$fio->{outDir}$dirM");
478 $fio->log("attribWrite(dir=$d) -> $fileName")
479 if ( $fio->{logLevel} >= 4 );
480 my $poolWrite = BackupPC::PoolWrite->new($fio->{bpc}, $fileName,
481 length($data), $fio->{xfer}{compress});
482 $poolWrite->write(\$data);
483 $fio->processClose($poolWrite, $fio->{attrib}{$d}->fileName($dirM),
486 delete($fio->{attrib}{$d});
491 my($fio, $poolWrite, $fileName, $origSize, $doStats) = @_;
492 my($exists, $digest, $outSize, $errs) = $poolWrite->close;
494 $fileName =~ s{^/+}{};
495 $fio->log(@$errs) if ( defined($errs) && @$errs );
497 $fio->{stats}{TotalFileCnt}++;
498 $fio->{stats}{TotalFileSize} += $origSize;
502 $fio->{stats}{ExistFileCnt}++;
503 $fio->{stats}{ExistFileSize} += $origSize;
504 $fio->{stats}{ExistFileCompSize} += $outSize;
506 } elsif ( $outSize > 0 ) {
507 my $fh = $fio->{newFilesFH};
508 print($fh "$digest $origSize $fileName\n") if ( defined($fh) );
510 return $exists && $origSize > 0;
517 return $fio->{stats};
521 # Make a given directory. Returns non-zero on error.
526 my $name = $1 if ( $f->{name} =~ /(.*)/ );
529 if ( $name eq "." ) {
530 $path = $fio->{outDirSh};
532 $path = $fio->{outDirSh} . $fio->{bpc}->fileNameMangle($name);
534 $fio->logFileAction("create", $f) if ( $fio->{logLevel} >= 1 );
535 $fio->log("makePath($path, 0777)") if ( $fio->{logLevel} >= 5 );
536 $path = $1 if ( $path =~ /(.*)/ );
537 File::Path::mkpath($path, 0, 0777) if ( !-d $path );
538 return $fio->attribSet($f) if ( -d $path );
539 $fio->log("Can't create directory $path");
540 $fio->{stats}{errorCnt}++;
545 # Make a special file. Returns non-zero on error.
550 my $name = $1 if ( $f->{name} =~ /(.*)/ );
551 my $fNameM = $fio->{bpc}->fileNameMangle($name);
552 my $path = $fio->{outDirSh} . $fNameM;
553 my $attr = $fio->attribGet($f);
555 my $type = $fio->mode2type($f->{mode});
557 $fio->log("makeSpecial($path, $type, $f->{mode})")
558 if ( $fio->{logLevel} >= 5 );
559 if ( $type == BPC_FTYPE_CHARDEV || $type == BPC_FTYPE_BLOCKDEV ) {
560 my($major, $minor, $fh, $fileData);
562 $major = $f->{rdev} >> 8;
563 $minor = $f->{rdev} & 0xff;
564 $str = "$major,$minor";
565 } elsif ( ($f->{mode} & S_IFMT) == S_IFLNK ) {
569 # Now see if the file is different, or this is a full, in which
570 # case we create the new file.
575 || $attr->{type} != $fio->mode2type($f->{mode})
576 || $attr->{mtime} != $f->{mtime}
577 || $attr->{size} != $f->{size}
578 || $attr->{uid} != $f->{uid}
579 || $attr->{gid} != $f->{gid}
580 || $attr->{mode} != $f->{mode}
581 || !defined($fh = BackupPC::FileZIO->open($attr->{fullPath}, 0,
583 || $fh->read(\$fileData, length($str) + 1) != length($str)
584 || $fileData ne $str ) {
585 $fh->close if ( defined($fh) );
586 $fh = BackupPC::PoolWrite->new($fio->{bpc}, $path,
587 length($str), $fio->{xfer}{compress});
589 my $exist = $fio->processClose($fh, "$fio->{shareM}/$fNameM",
591 $fio->logFileAction($exist ? "pool" : "create", $f)
592 if ( $fio->{logLevel} >= 1 );
593 return $fio->attribSet($f);
595 $fio->logFileAction("skip", $f) if ( $fio->{logLevel} >= 2 );
597 $fh->close if ( defined($fh) );
602 my($fio, $path) = @_;
604 $fio->log("Unexpected call BackupPC::Xfer::RsyncFileIO->unlink($path)");
608 # Default log handler
614 print(STDERR $str, "\n");
618 # Handle one or more log messages
622 my($fio, @logStr) = @_;
624 foreach my $str ( @logStr ) {
625 next if ( $str eq "" );
626 $fio->{logHandler}($str);
631 # Generate a log file message for a completed file
635 my($fio, $action, $f) = @_;
636 my $owner = "$f->{uid}/$f->{gid}";
637 my $type = (("", "p", "c", "", "d", "", "b", "", "", "", "l", "", "s"))
638 [($f->{mode} & S_IFMT) >> 12];
640 $fio->log(sprintf(" %-6s %1s%4o %9s %11.0f %s",
650 # If there is a partial and we are doing a full, we do an incremental
651 # against the partial and a full against the rest. This subroutine
652 # is how we tell File::RsyncP which files to ignore attributes on
653 # (ie: against the partial dump we do consider the attributes, but
654 # otherwise we ignore attributes).
660 return if ( !defined($fio->{partialNum}) );
661 my($attr, $isPartial) = $fio->attribGetWhere($f);
662 $fio->log("$f->{name}: just checking attributes from partial")
663 if ( $isPartial && $fio->{logLevel} >= 5 );
668 # This is called by File::RsyncP when a file is skipped because the
673 my($fio, $f, $attr) = @_;
676 # Unless this is a partial, this is normal so ignore it.
678 return if ( !defined($fio->{partialNum}) );
680 $fio->log("$f->{name}: skipped in partial; adding link")
681 if ( $fio->{logLevel} >= 5 );
682 $fio->{rxLocalAttr} = $attr;
684 $fio->{rxSize} = $attr->{size};
685 delete($fio->{rxInFd});
686 delete($fio->{rxOutFd});
687 delete($fio->{rxDigest});
688 delete($fio->{rxInData});
689 return $fio->fileDeltaRxDone();
693 # Start receive of file deltas for a particular file.
697 my($fio, $f, $cnt, $size, $remainder) = @_;
699 $fio->{rxFile} = $f; # remote file attributes
700 $fio->{rxLocalAttr} = $fio->attribGet($f); # local file attributes
701 $fio->{rxBlkCnt} = $cnt; # how many blocks we will receive
702 $fio->{rxBlkSize} = $size; # block size
703 $fio->{rxRemainder} = $remainder; # size of the last block
704 $fio->{rxMatchBlk} = 0; # current start of match
705 $fio->{rxMatchNext} = 0; # current next block of match
706 $fio->{rxSize} = 0; # size of received file
707 my $rxSize = $cnt > 0 ? ($cnt - 1) * $size + $remainder : 0;
708 if ( $fio->{rxFile}{size} != $rxSize ) {
709 $fio->{rxMatchBlk} = undef; # size different, so no file match
710 $fio->log("$fio->{rxFile}{name}: size doesn't match"
711 . " ($fio->{rxFile}{size} vs $rxSize)")
712 if ( $fio->{logLevel} >= 5 );
714 delete($fio->{rxInFd});
715 delete($fio->{rxOutFd});
716 delete($fio->{rxDigest});
717 delete($fio->{rxInData});
721 # Process the next file delta for the current file. Returns 0 if ok,
722 # -1 if not. Must be called with either a block number, $blk, or new data,
723 # $newData, (not both) defined.
727 my($fio, $blk, $newData) = @_;
729 if ( defined($blk) ) {
730 if ( defined($fio->{rxMatchBlk}) && $fio->{rxMatchNext} == $blk ) {
732 # got the next block in order; just keep track.
734 $fio->{rxMatchNext}++;
738 my $newDataLen = length($newData);
739 $fio->log("$fio->{rxFile}{name}: blk=$blk, newData=$newDataLen, rxMatchBlk=$fio->{rxMatchBlk}, rxMatchNext=$fio->{rxMatchNext}")
740 if ( $fio->{logLevel} >= 8 );
741 if ( !defined($fio->{rxOutFd}) ) {
743 # maybe the file has no changes
745 if ( $fio->{rxMatchNext} == $fio->{rxBlkCnt}
746 && !defined($blk) && !defined($newData) ) {
747 #$fio->log("$fio->{rxFile}{name}: file is unchanged");
748 # if ( $fio->{logLevel} >= 8 );
753 # need to open an output file where we will build the
756 $fio->{rxFile}{name} =~ /(.*)/;
757 my $rxOutFileRel = "$fio->{shareM}/" . $fio->{bpc}->fileNameMangle($1);
758 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
759 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
760 $rxOutFile, $fio->{rxFile}{size},
761 $fio->{xfer}{compress});
762 $fio->log("$fio->{rxFile}{name}: opening output file $rxOutFile")
763 if ( $fio->{logLevel} >= 9 );
764 $fio->{rxOutFile} = $rxOutFile;
765 $fio->{rxOutFileRel} = $rxOutFileRel;
766 $fio->{rxDigest} = File::RsyncP::Digest->new;
767 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
769 if ( defined($fio->{rxMatchBlk})
770 && $fio->{rxMatchBlk} != $fio->{rxMatchNext} ) {
772 # Need to copy the sequence of blocks that matched. If the file
773 # is compressed we need to make a copy of the uncompressed file,
774 # since the compressed file is not seekable. Future optimizations
775 # could include only creating an uncompressed copy if the matching
776 # blocks were not monotonic, and to only do this if there are
777 # matching blocks (eg, maybe the entire file is new).
779 my $attr = $fio->{rxLocalAttr};
781 if ( !defined($fio->{rxInFd}) && !defined($fio->{rxInData}) ) {
782 if ( $attr->{compress} ) {
783 if ( !defined($fh = BackupPC::FileZIO->open(
786 $attr->{compress})) ) {
787 $fio->log("Can't open $attr->{fullPath}");
788 $fio->{stats}{errorCnt}++;
791 if ( $attr->{size} < 16 * 1024 * 1024 ) {
793 # Cache the entire old file if it is less than 16MB
796 $fio->{rxInData} = "";
797 while ( $fh->read(\$data, 16 * 1024 * 1024) > 0 ) {
798 $fio->{rxInData} .= $data;
800 $fio->log("$attr->{fullPath}: cached all $attr->{size}"
802 if ( $fio->{logLevel} >= 9 );
805 # Create and write a temporary output file
807 unlink("$fio->{outDirSh}RStmp")
808 if ( -f "$fio->{outDirSh}RStmp" );
809 if ( open(F, "+>", "$fio->{outDirSh}RStmp") ) {
813 while ( $fh->read(\$data, 1024 * 1024) > 0 ) {
814 if ( syswrite(F, $data) != length($data) ) {
815 $fio->log(sprintf("Can't write len=%d to %s",
816 length($data) , "$fio->{outDirSh}RStmp"));
818 $fio->{stats}{errorCnt}++;
821 $byteCnt += length($data);
824 $fio->{rxInName} = "$fio->{outDirSh}RStmp";
825 sysseek($fio->{rxInFd}, 0, 0);
826 $fio->log("$attr->{fullPath}: copied $byteCnt,"
827 . "$attr->{size} bytes to $fio->{rxInName}")
828 if ( $fio->{logLevel} >= 9 );
830 $fio->log("Unable to open $fio->{outDirSh}RStmp");
832 $fio->{stats}{errorCnt}++;
838 if ( open(F, "<", $attr->{fullPath}) ) {
841 $fio->{rxInName} = $attr->{fullPath};
843 $fio->log("Unable to open $attr->{fullPath}");
844 $fio->{stats}{errorCnt}++;
849 my $lastBlk = $fio->{rxMatchNext} - 1;
850 $fio->log("$fio->{rxFile}{name}: writing blocks $fio->{rxMatchBlk}.."
852 if ( $fio->{logLevel} >= 9 );
853 my $seekPosn = $fio->{rxMatchBlk} * $fio->{rxBlkSize};
854 if ( defined($fio->{rxInFd})
855 && !sysseek($fio->{rxInFd}, $seekPosn, 0) ) {
856 $fio->log("Unable to seek $attr->{rxInName} to $seekPosn");
857 $fio->{stats}{errorCnt}++;
860 my $cnt = $fio->{rxMatchNext} - $fio->{rxMatchBlk};
861 my($thisCnt, $len, $data);
862 for ( my $i = 0 ; $i < $cnt ; $i += $thisCnt ) {
863 $thisCnt = $cnt - $i;
864 $thisCnt = 512 if ( $thisCnt > 512 );
865 if ( $fio->{rxMatchBlk} + $i + $thisCnt == $fio->{rxBlkCnt} ) {
866 $len = ($thisCnt - 1) * $fio->{rxBlkSize} + $fio->{rxRemainder};
868 $len = $thisCnt * $fio->{rxBlkSize};
870 if ( defined($fio->{rxInData}) ) {
871 $data = substr($fio->{rxInData}, $seekPosn, $len);
874 my $got = sysread($fio->{rxInFd}, $data, $len);
875 if ( $got != $len ) {
876 my $inFileSize = -s $fio->{rxInName};
877 $fio->log("Unable to read $len bytes from $fio->{rxInName}"
878 . " got=$got, seekPosn=$seekPosn"
879 . " ($i,$thisCnt,$fio->{rxBlkCnt},$inFileSize"
880 . ",$attr->{size})");
881 $fio->{stats}{errorCnt}++;
886 $fio->{rxOutFd}->write(\$data);
887 $fio->{rxDigest}->add($data);
888 $fio->{rxSize} += length($data);
890 $fio->{rxMatchBlk} = undef;
892 if ( defined($blk) ) {
894 # Remember the new block number
896 $fio->{rxMatchBlk} = $blk;
897 $fio->{rxMatchNext} = $blk + 1;
899 if ( defined($newData) ) {
901 # Write the new chunk
903 my $len = length($newData);
904 $fio->log("$fio->{rxFile}{name}: writing $len bytes new data")
905 if ( $fio->{logLevel} >= 9 );
906 $fio->{rxOutFd}->write(\$newData);
907 $fio->{rxDigest}->add($newData);
908 $fio->{rxSize} += length($newData);
913 # Finish up the current receive file. Returns undef if ok, -1 if not.
914 # Returns 1 if the md4 digest doesn't match.
918 my($fio, $md4, $phase) = @_;
919 my $name = $1 if ( $fio->{rxFile}{name} =~ /(.*)/ );
922 close($fio->{rxInFd}) if ( defined($fio->{rxInFd}) );
923 unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" );
926 # Check the final md4 digest
928 if ( defined($md4) ) {
930 if ( !defined($fio->{rxDigest}) ) {
932 # File was exact match, but we still need to verify the
933 # MD4 checksum. Compute the md4 digest (or fetch the
936 if ( defined(my $attr = $fio->{rxLocalAttr}) ) {
938 # block size doesn't matter: we're only going to
939 # fetch the md4 file digest, not the block digests.
941 my($err, $csum, $blkSize)
942 = BackupPC::Xfer::RsyncDigest->digestStart(
943 $attr->{fullPath}, $attr->{size},
944 0, 2048, $fio->{checksumSeed}, 1,
945 $attr->{compress}, 1);
947 $fio->log("Can't open $attr->{fullPath} for MD4"
948 . " check (err=$err, $name)");
949 $fio->{stats}{errorCnt}++;
951 if ( $fio->{logLevel} >= 5 ) {
952 my($isCached, $invalid) = $csum->isCached;
953 $fio->log("MD4 $attr->{fullPath} cache = $isCached,"
954 . " invalid = $invalid");
956 $newDigest = $csum->digestEnd;
958 $fio->{rxSize} = $attr->{size};
961 # Empty file; just create an empty file digest
963 $fio->{rxDigest} = File::RsyncP::Digest->new;
964 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
965 $newDigest = $fio->{rxDigest}->digest;
967 $fio->log("$name got exact match") if ( $fio->{logLevel} >= 5 );
969 $newDigest = $fio->{rxDigest}->digest;
971 if ( $fio->{logLevel} >= 3 ) {
972 my $md4Str = unpack("H*", $md4);
973 my $newStr = unpack("H*", $newDigest);
974 $fio->log("$name got digests $md4Str vs $newStr")
976 if ( $md4 ne $newDigest ) {
978 $fio->log("$name: fatal error: md4 doesn't match on retry;"
981 $fio->log("$name: md4 doesn't match: will retry in phase 1;"
984 $fio->{stats}{errorCnt}++;
985 if ( defined($fio->{rxOutFd}) ) {
986 $fio->{rxOutFd}->close;
987 unlink($fio->{rxOutFile});
989 delete($fio->{rxFile});
990 delete($fio->{rxOutFile});
996 # One special case is an empty file: if the file size is
997 # zero we need to open the output file to create it.
999 if ( $fio->{rxSize} == 0 ) {
1000 my $rxOutFileRel = "$fio->{shareM}/"
1001 . $fio->{bpc}->fileNameMangle($name);
1002 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
1003 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
1004 $rxOutFile, $fio->{rxSize},
1005 $fio->{xfer}{compress});
1007 if ( !defined($fio->{rxOutFd}) ) {
1009 # No output file, meaning original was an exact match.
1011 $fio->log("$name: nothing to do")
1012 if ( $fio->{logLevel} >= 5 );
1013 my $attr = $fio->{rxLocalAttr};
1014 my $f = $fio->{rxFile};
1015 $fio->logFileAction("same", $f) if ( $fio->{logLevel} >= 1 );
1017 || $attr->{type} != $f->{type}
1018 || $attr->{mtime} != $f->{mtime}
1019 || $attr->{size} != $f->{size}
1020 || $attr->{gid} != $f->{gid}
1021 || $attr->{mode} != $f->{mode} ) {
1023 # In the full case, or if the attributes are different,
1024 # we need to make a link from the previous file and
1025 # set the attributes.
1027 my $rxOutFile = $fio->{outDirSh}
1028 . $fio->{bpc}->fileNameMangle($name);
1029 if ( !link($attr->{fullPath}, $rxOutFile) ) {
1030 $fio->log("Unable to link $attr->{fullPath} to $rxOutFile");
1031 $fio->{stats}{errorCnt}++;
1035 # Cumulate the stats
1037 $fio->{stats}{TotalFileCnt}++;
1038 $fio->{stats}{TotalFileSize} += $fio->{rxSize};
1039 $fio->{stats}{ExistFileCnt}++;
1040 $fio->{stats}{ExistFileSize} += $fio->{rxSize};
1041 $fio->{stats}{ExistFileCompSize} += -s $rxOutFile;
1042 $fio->{rxFile}{size} = $fio->{rxSize};
1043 $ret = $fio->attribSet($fio->{rxFile});
1047 my $exist = $fio->processClose($fio->{rxOutFd},
1048 $fio->{rxOutFileRel},
1050 $fio->logFileAction($exist ? "pool" : "create", $fio->{rxFile})
1051 if ( $fio->{logLevel} >= 1 );
1052 $fio->{rxFile}{size} = $fio->{rxSize};
1053 $ret = $fio->attribSet($fio->{rxFile});
1055 delete($fio->{rxDigest});
1056 delete($fio->{rxInData});
1057 delete($fio->{rxFile});
1058 delete($fio->{rxOutFile});
1063 # Callback function for BackupPC::View->find. Note the order of the
1064 # first two arguments.
1068 my($a, $fio, $fList, $outputFunc) = @_;
1069 my $name = $a->{relPath};
1071 my $type = $fio->mode2type($a->{mode});
1072 my $extraAttribs = {};
1074 $n =~ s/^\Q$fio->{xfer}{pathHdrSrc}//;
1075 $fio->log("Sending $name (remote=$n)") if ( $fio->{logLevel} >= 4 );
1076 if ( $type == BPC_FTYPE_CHARDEV
1077 || $type == BPC_FTYPE_BLOCKDEV
1078 || $type == BPC_FTYPE_SYMLINK ) {
1079 my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0, $a->{compress});
1081 if ( defined($fh) ) {
1082 $rdSize = $fh->read(\$str, $a->{size} + 1024);
1083 if ( $type == BPC_FTYPE_SYMLINK ) {
1085 # Reconstruct symbolic link
1087 $extraAttribs = { link => $str };
1088 if ( $rdSize != $a->{size} ) {
1090 $fio->log("$name: can't read exactly $a->{size} bytes");
1091 $fio->{stats}{errorCnt}++;
1093 } elsif ( $str =~ /(\d*),(\d*)/ ) {
1095 # Reconstruct char or block special major/minor device num
1097 # Note: char/block devices have $a->{size} = 0, so we
1098 # can't do an error check on $rdSize.
1100 $extraAttribs = { rdev => $1 * 256 + $2 };
1102 $fio->log("$name: unexpected special file contents $str");
1103 $fio->{stats}{errorCnt}++;
1108 $fio->log("$name: can't open");
1109 $fio->{stats}{errorCnt}++;
1114 #dev => 0, # later, when we support hardlinks
1115 #inode => 0, # later, when we support hardlinks
1119 mtime => $a->{mtime},
1124 $f->{name} = "$fio->{xfer}{pathHdrDest}/$f->{name}";
1125 $f->{name} =~ s{//+}{/}g;
1126 $fio->logFileAction("restore", $f) if ( $fio->{logLevel} >= 1 );
1127 &$outputFunc($fList->encodeData);
1131 if ( $type != BPC_FTYPE_DIR ) {
1132 $fio->{stats}{TotalFileCnt}++;
1133 $fio->{stats}{TotalFileSize} += $a->{size};
1139 my($fio, $flist, $outputFunc) = @_;
1142 # Populate the file list with the files requested by the user.
1143 # Since some might be directories so we call BackupPC::View::find.
1145 $fio->log("fileListSend: sending file list: "
1146 . join(" ", @{$fio->{fileList}})) if ( $fio->{logLevel} >= 4 );
1147 foreach my $name ( @{$fio->{fileList}} ) {
1148 $fio->{view}->find($fio->{xfer}{bkupSrcNum},
1149 $fio->{xfer}{bkupSrcShare},
1151 \&fileListEltSend, $fio, $flist, $outputFunc);
1157 my($fio, $isChild) = @_;
1160 # If we are aborting early, remove the last file since
1161 # it was not complete
1163 if ( $isChild && defined($fio->{rxFile}) ) {
1164 unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" );
1165 if ( defined($fio->{rxFile}) ) {
1166 unlink($fio->{rxOutFile});
1167 $fio->log("finish: removing in-process file $fio->{rxFile}{name}");
1172 # Flush the attributes if this is the child
1174 $fio->attribWrite(undef) if ( $isChild );
1180 # join('',@_), kill 0;