1 #============================================================= -*-perl-*-
8 # Craig Barratt <cbarratt@users.sourceforge.net>
11 # Copyright (C) 2002 Craig Barratt
13 #========================================================================
15 # Version 2.0.0beta2, released 13 Apr 2003.
17 # See http://backuppc.sourceforge.net.
19 #========================================================================
21 package BackupPC::Xfer::RsyncFileIO;
25 use BackupPC::Attrib qw(:all);
27 use BackupPC::PoolWrite;
28 use BackupPC::PoolWrite;
31 use constant S_IFMT => 0170000; # type of file
32 use constant S_IFDIR => 0040000; # directory
33 use constant S_IFCHR => 0020000; # character special
34 use constant S_IFBLK => 0060000; # block special
35 use constant S_IFREG => 0100000; # regular
36 use constant S_IFLNK => 0120000; # symbolic link
37 use constant S_IFSOCK => 0140000; # socket
38 use constant S_IFIFO => 0010000; # fifo
40 use vars qw( $RsyncLibOK );
43 eval "use File::RsyncP::Digest";
46 # Rsync module doesn't exist.
56 my($class, $options) = @_;
58 return if ( !$RsyncLibOK );
63 digest => File::RsyncP::Digest->new,
66 logHandler => \&logHandler,
73 ExistFileCompSize => 0,
78 $fio->{shareM} = $fio->{bpc}->fileNameEltMangle($fio->{share});
79 $fio->{outDir} = "$fio->{xfer}{outDir}/new/";
80 $fio->{outDirSh} = "$fio->{outDir}/$fio->{shareM}/";
81 $fio->{view} = BackupPC::View->new($fio->{bpc}, $fio->{client},
83 $fio->{full} = $fio->{xfer}{type} eq "full" ? 1 : 0;
84 $fio->{newFilesFH} = $fio->{xfer}{newFilesFH};
90 my($fio, $value) = @_;
92 $fio->{blockSize} = $value if ( defined($value) );
93 return $fio->{blockSize};
99 $fio->{logHandler} = $sub;
103 # Setup rsync checksum computation for the given file.
107 my($fio, $f, $needMD4) = @_;
109 my $attr = $fio->attribGet($f);
111 $fio->csumEnd if ( defined($fio->{fh}) );
112 return if ( $attr->{type} != BPC_FTYPE_FILE );
113 if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath},
115 $attr->{compress})) ) {
116 $fio->log("Can't open $attr->{fullPath} (name=$f->{name})");
117 $fio->{stats}{errorCnt}++;
121 $fio->{csumDigest} = File::RsyncP::Digest->new;
122 $fio->{csumDigest}->add(pack("V", $fio->{checksumSeed}));
124 delete($fio->{csumDigest});
130 my($fio, $num, $csumLen, $blockSize) = @_;
136 return if ( !defined($fio->{fh}) );
137 if ( $fio->{fh}->read(\$fileData, $blockSize * $num) <= 0 ) {
138 $fio->log("$fio->{file}{name}: csumGet is at EOF - zero padding");
139 $fio->{stats}{errorCnt}++;
140 $fileData = pack("c", 0) x ($blockSize * $num);
142 $fio->{csumDigest}->add($fileData) if ( defined($fio->{csumDigest}) );
143 $fio->log(sprintf("%s: getting csum ($num,$csumLen,%d,0x%x)",
146 $fio->{checksumSeed}))
147 if ( $fio->{logLevel} >= 9 );
148 return $fio->{digest}->blockDigest($fileData, $blockSize,
149 $csumLen, $fio->{checksumSeed});
156 return if ( !defined($fio->{fh}) );
158 # make sure we read the entire file for the file MD4 digest
160 if ( defined($fio->{csumDigest}) ) {
162 while ( $fio->{fh}->read(\$fileData, 65536) > 0 ) {
163 $fio->{csumDigest}->add($fileData);
168 return $fio->{csumDigest}->digest if ( defined($fio->{csumDigest}) );
175 my $attr = $fio->attribGet($f);
177 $fio->readEnd if ( defined($fio->{fh}) );
178 if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath},
180 $attr->{compress})) ) {
181 $fio->log("Can't open $attr->{fullPath} (name=$f->{name})");
182 $fio->{stats}{errorCnt}++;
185 $fio->log("$f->{name}: opened for read") if ( $fio->{logLevel} >= 4 );
194 return if ( !defined($fio->{fh}) );
195 if ( $fio->{fh}->read(\$fileData, $num) <= 0 ) {
196 return $fio->readEnd;
198 $fio->log(sprintf("read returns %d bytes", length($fileData)))
199 if ( $fio->{logLevel} >= 8 );
207 return if ( !defined($fio->{fh}) );
209 $fio->log("closing $fio->{file}{name})") if ( $fio->{logLevel} >= 8 );
216 my($fio, $checksumSeed) = @_;
218 $fio->{checksumSeed} = $checksumSeed;
223 my($fio, $localDir, $remoteDir) = @_;
225 $fio->{localDir} = $localDir;
226 $fio->{remoteDir} = $remoteDir;
231 my($fio, $share, $dir) = @_;
234 #$fio->log("viewCacheDir($share, $dir)");
235 if ( !defined($share) ) {
236 $share = $fio->{share};
237 $shareM = $fio->{shareM};
239 $shareM = $fio->{bpc}->fileNameEltMangle($share);
241 $shareM = "$shareM/$dir" if ( $dir ne "" );
242 return if ( defined($fio->{viewCache}{$shareM}) );
244 # purge old cache entries (ie: those that don't match the
245 # first part of $dir).
247 foreach my $d ( keys(%{$fio->{viewCache}}) ) {
248 delete($fio->{viewCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
251 # fetch new directory attributes
253 $fio->{viewCache}{$shareM}
254 = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir);
260 my($dir, $fname, $share, $shareM);
263 $fname = "$fio->{xfer}{pathHdrSrc}/$fname"
264 if ( defined($fio->{xfer}{pathHdrSrc}) );
265 $fname =~ s{//+}{/}g;
266 if ( $fname =~ m{(.*)/(.*)} ) {
267 $shareM = $fio->{shareM};
270 } elsif ( $fname ne "." ) {
271 $shareM = $fio->{shareM};
277 $fname = $fio->{share};
279 $fio->viewCacheDir($share, $dir);
280 $shareM .= "/$dir" if ( $dir ne "" );
281 return $fio->{viewCache}{$shareM}{$fname};
286 my($fio, $mode) = @_;
288 if ( ($mode & S_IFMT) == S_IFREG ) {
289 return BPC_FTYPE_FILE;
290 } elsif ( ($mode & S_IFMT) == S_IFDIR ) {
291 return BPC_FTYPE_DIR;
292 } elsif ( ($mode & S_IFMT) == S_IFLNK ) {
293 return BPC_FTYPE_SYMLINK;
294 } elsif ( ($mode & S_IFMT) == S_IFCHR ) {
295 return BPC_FTYPE_CHARDEV;
296 } elsif ( ($mode & S_IFMT) == S_IFBLK ) {
297 return BPC_FTYPE_BLOCKDEV;
298 } elsif ( ($mode & S_IFMT) == S_IFIFO ) {
299 return BPC_FTYPE_FIFO;
300 } elsif ( ($mode & S_IFMT) == S_IFSOCK ) {
301 return BPC_FTYPE_SOCKET;
303 return BPC_FTYPE_UNKNOWN;
308 # Set the attributes for a file. Returns non-zero on error.
312 my($fio, $f, $placeHolder) = @_;
315 if ( $f->{name} =~ m{(.*)/(.*)} ) {
317 $dir = "$fio->{shareM}/" . $1;
318 } elsif ( $f->{name} eq "." ) {
320 $file = $fio->{share};
322 $dir = $fio->{shareM};
326 if ( !defined($fio->{attribLastDir}) || $fio->{attribLastDir} ne $dir ) {
328 # Flush any directories that don't match the first part
329 # of the new directory
331 foreach my $d ( keys(%{$fio->{attrib}}) ) {
332 next if ( $d eq "" || "$dir/" =~ m{^\Q$d/} );
333 $fio->attribWrite($d);
335 $fio->{attribLastDir} = $dir;
337 if ( !exists($fio->{attrib}{$dir}) ) {
338 $fio->{attrib}{$dir} = BackupPC::Attrib->new({
339 compress => $fio->{xfer}{compress},
341 my $path = $fio->{outDir} . $dir;
342 if ( -f $fio->{attrib}{$dir}->fileName($path)
343 && !$fio->{attrib}{$dir}->read($path) ) {
344 $fio->log(sprintf("Unable to read attribute file %s",
345 $fio->{attrib}{$dir}->fileName($path)));
348 $fio->log("attribSet(dir=$dir, file=$file)") if ( $fio->{logLevel} >= 4 );
350 $fio->{attrib}{$dir}->set($file, {
351 type => $fio->mode2type($f->{mode}),
355 size => $placeHolder ? -1 : $f->{size},
356 mtime => $f->{mtime},
366 if ( !defined($d) ) {
368 # flush all entries (in reverse order)
370 foreach $d ( sort({$b cmp $a} keys(%{$fio->{attrib}})) ) {
371 $fio->attribWrite($d);
375 return if ( !defined($fio->{attrib}{$d}) );
377 # Set deleted files in the attributes. Any file in the view
378 # that doesn't have attributes is deleted. All files sent by
379 # rsync have attributes temporarily set so we can do deletion
380 # detection. We also prune these temporary attributes.
386 $dir = $1 if ( $d =~ m{.+?/(.*)} );
387 $fio->viewCacheDir(undef, $dir);
388 ##print("attribWrite $d,$dir\n");
389 ##$Data::Dumper::Indent = 1;
390 ##$fio->log("attribWrite $d,$dir");
391 ##$fio->log("viewCacheLogKeys = ", keys(%{$fio->{viewCache}}));
392 ##$fio->log("attribKeys = ", keys(%{$fio->{attrib}}));
393 ##print "viewCache = ", Dumper($fio->{attrib});
394 ##print "attrib = ", Dumper($fio->{attrib});
395 if ( defined($fio->{viewCache}{$d}) ) {
396 foreach my $f ( keys(%{$fio->{viewCache}{$d}}) ) {
398 $name = "$1/$name" if ( $d =~ m{.*?/(.*)} );
399 if ( defined(my $a = $fio->{attrib}{$d}->get($f)) ) {
401 # delete temporary attributes (skipped files)
403 if ( $a->{size} < 0 ) {
404 $fio->{attrib}{$d}->set($f, undef);
405 $fio->logFileAction("skip", {
406 %{$fio->{viewCache}{$d}{$f}},
408 }) if ( $fio->{logLevel} >= 2 );
411 ##print("Delete file $f\n");
412 $fio->logFileAction("delete", {
413 %{$fio->{viewCache}{$d}{$f}},
415 }) if ( $fio->{logLevel} >= 1 );
416 $fio->{attrib}{$d}->set($f, {
417 type => BPC_FTYPE_DELETED,
428 if ( $fio->{attrib}{$d}->fileCount ) {
429 my $data = $fio->{attrib}{$d}->writeData;
432 $dirM = $1 . "/" . $fio->{bpc}->fileNameMangle($2)
433 if ( $dirM =~ m{(.*?)/(.*)} );
434 my $fileName = $fio->{attrib}{$d}->fileName("$fio->{outDir}$dirM");
435 $fio->log("attribWrite(dir=$d) -> $fileName")
436 if ( $fio->{logLevel} >= 4 );
437 my $poolWrite = BackupPC::PoolWrite->new($fio->{bpc}, $fileName,
438 length($data), $fio->{xfer}{compress});
439 $poolWrite->write(\$data);
440 $fio->processClose($poolWrite, $fio->{attrib}{$d}->fileName($dirM),
443 delete($fio->{attrib}{$d});
448 my($fio, $poolWrite, $fileName, $origSize, $doStats) = @_;
449 my($exists, $digest, $outSize, $errs) = $poolWrite->close;
451 $fileName =~ s{^/+}{};
452 $fio->log(@$errs) if ( defined($errs) && @$errs );
454 $fio->{stats}{TotalFileCnt}++;
455 $fio->{stats}{TotalFileSize} += $origSize;
459 $fio->{stats}{ExistFileCnt}++;
460 $fio->{stats}{ExistFileSize} += $origSize;
461 $fio->{stats}{ExistFileCompSize} += $outSize;
463 } elsif ( $outSize > 0 ) {
464 my $fh = $fio->{newFilesFH};
465 print($fh "$digest $origSize $fileName\n") if ( defined($fh) );
467 return $exists && $origSize > 0;
474 return $fio->{stats};
478 # Make a given directory. Returns non-zero on error.
483 my $name = $1 if ( $f->{name} =~ /(.*)/ );
486 if ( $name eq "." ) {
487 $path = $fio->{outDirSh};
489 $path = $fio->{outDirSh} . $fio->{bpc}->fileNameMangle($name);
491 $fio->logFileAction("create", $f) if ( $fio->{logLevel} >= 1 );
492 $fio->log("makePath($path, 0777)") if ( $fio->{logLevel} >= 5 );
493 $path = $1 if ( $path =~ /(.*)/ );
494 File::Path::mkpath($path, 0, 0777) if ( !-d $path );
495 return $fio->attribSet($f) if ( -d $path );
496 $fio->log("Can't create directory $path");
497 $fio->{stats}{errorCnt}++;
502 # Make a special file. Returns non-zero on error.
507 my $name = $1 if ( $f->{name} =~ /(.*)/ );
508 my $fNameM = $fio->{bpc}->fileNameMangle($name);
509 my $path = $fio->{outDirSh} . $fNameM;
510 my $attr = $fio->attribGet($f);
512 my $type = $fio->mode2type($f->{mode});
514 $fio->log("makeSpecial($path, $type, $f->{mode})")
515 if ( $fio->{logLevel} >= 5 );
516 if ( $type == BPC_FTYPE_CHARDEV || $type == BPC_FTYPE_BLOCKDEV ) {
517 my($major, $minor, $fh, $fileData);
519 $major = $f->{rdev} >> 8;
520 $minor = $f->{rdev} & 0xff;
521 $str = "$major,$minor";
522 } elsif ( ($f->{mode} & S_IFMT) == S_IFLNK ) {
526 # Now see if the file is different, or this is a full, in which
527 # case we create the new file.
532 || $attr->{type} != $fio->mode2type($f->{mode})
533 || $attr->{mtime} != $f->{mtime}
534 || $attr->{size} != $f->{size}
535 || $attr->{uid} != $f->{uid}
536 || $attr->{gid} != $f->{gid}
537 || $attr->{mode} != $f->{mode}
538 || !defined($fh = BackupPC::FileZIO->open($attr->{fullPath}, 0,
540 || $fh->read(\$fileData, length($str) + 1) != length($str)
541 || $fileData ne $str ) {
542 $fh->close if ( defined($fh) );
543 $fh = BackupPC::PoolWrite->new($fio->{bpc}, $path,
544 length($str), $fio->{xfer}{compress});
546 my $exist = $fio->processClose($fh, "$fio->{shareM}/$fNameM",
548 $fio->logFileAction($exist ? "pool" : "create", $f)
549 if ( $fio->{logLevel} >= 1 );
550 return $fio->attribSet($f);
552 $fio->logFileAction("skip", $f) if ( $fio->{logLevel} >= 2 );
554 $fh->close if ( defined($fh) );
559 my($fio, $path) = @_;
561 $fio->log("Unexpected call BackupPC::Xfer::RsyncFileIO->unlink($path)");
565 # Default log handler
571 print(STDERR $str, "\n");
575 # Handle one or more log messages
579 my($fio, @logStr) = @_;
581 foreach my $str ( @logStr ) {
582 next if ( $str eq "" );
583 $fio->{logHandler}($str);
588 # Generate a log file message for a completed file
592 my($fio, $action, $f) = @_;
593 my $owner = "$f->{uid}/$f->{gid}";
594 my $type = (("", "p", "c", "", "d", "", "b", "", "", "", "l", "", "s"))
595 [($f->{mode} & S_IFMT) >> 12];
597 $fio->log(sprintf(" %-6s %1s%4o %9s %11.0f %s",
607 # Later we'll use this function to complete a prior unfinished dump.
608 # We'll do an incremental on the part we have already, and then a
609 # full or incremental against the rest.
617 # Start receive of file deltas for a particular file.
621 my($fio, $f, $cnt, $size, $remainder) = @_;
623 $fio->{rxFile} = $f; # remote file attributes
624 $fio->{rxLocalAttr} = $fio->attribGet($f); # local file attributes
625 $fio->{rxBlkCnt} = $cnt; # how many blocks we will receive
626 $fio->{rxBlkSize} = $size; # block size
627 $fio->{rxRemainder} = $remainder; # size of the last block
628 $fio->{rxMatchBlk} = 0; # current start of match
629 $fio->{rxMatchNext} = 0; # current next block of match
630 $fio->{rxSize} = 0; # size of received file
631 my $rxSize = $cnt > 0 ? ($cnt - 1) * $size + $remainder : 0;
632 if ( $fio->{rxFile}{size} != $rxSize ) {
633 $fio->{rxMatchBlk} = undef; # size different, so no file match
634 $fio->log("$fio->{rxFile}{name}: size doesn't match"
635 . " ($fio->{rxFile}{size} vs $rxSize)")
636 if ( $fio->{logLevel} >= 5 );
638 delete($fio->{rxInFd});
639 delete($fio->{rxOutFd});
640 delete($fio->{rxDigest});
641 delete($fio->{rxInData});
645 # Process the next file delta for the current file. Returns 0 if ok,
646 # -1 if not. Must be called with either a block number, $blk, or new data,
647 # $newData, (not both) defined.
651 my($fio, $blk, $newData) = @_;
653 if ( defined($blk) ) {
654 if ( defined($fio->{rxMatchBlk}) && $fio->{rxMatchNext} == $blk ) {
656 # got the next block in order; just keep track.
658 $fio->{rxMatchNext}++;
662 my $newDataLen = length($newData);
663 $fio->log("$fio->{rxFile}{name}: blk=$blk, newData=$newDataLen, rxMatchBlk=$fio->{rxMatchBlk}, rxMatchNext=$fio->{rxMatchNext}")
664 if ( $fio->{logLevel} >= 8 );
665 if ( !defined($fio->{rxOutFd}) ) {
667 # maybe the file has no changes
669 if ( $fio->{rxMatchNext} == $fio->{rxBlkCnt}
670 && !defined($blk) && !defined($newData) ) {
671 #$fio->log("$fio->{rxFile}{name}: file is unchanged");
672 # if ( $fio->{logLevel} >= 8 );
677 # need to open an output file where we will build the
680 $fio->{rxFile}{name} =~ /(.*)/;
681 my $rxOutFileRel = "$fio->{shareM}/" . $fio->{bpc}->fileNameMangle($1);
682 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
683 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
684 $rxOutFile, $fio->{rxFile}{size},
685 $fio->{xfer}{compress});
686 $fio->log("$fio->{rxFile}{name}: opening output file $rxOutFile")
687 if ( $fio->{logLevel} >= 9 );
688 $fio->{rxOutFile} = $rxOutFile;
689 $fio->{rxOutFileRel} = $rxOutFileRel;
690 $fio->{rxDigest} = File::RsyncP::Digest->new;
691 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
693 if ( defined($fio->{rxMatchBlk})
694 && $fio->{rxMatchBlk} != $fio->{rxMatchNext} ) {
696 # Need to copy the sequence of blocks that matched. If the file
697 # is compressed we need to make a copy of the uncompressed file,
698 # since the compressed file is not seekable. Future optimizations
699 # would be to keep the uncompressed file in memory (eg, up to say
700 # 10MB), only create an uncompressed copy if the matching
701 # blocks were not monotonic, and to only do this if there are
702 # matching blocks (eg, maybe the entire file is new).
704 my $attr = $fio->{rxLocalAttr};
706 if ( !defined($fio->{rxInFd}) && !defined($fio->{rxInData}) ) {
707 if ( $attr->{compress} ) {
708 if ( !defined($fh = BackupPC::FileZIO->open(
711 $attr->{compress})) ) {
712 $fio->log("Can't open $attr->{fullPath}");
713 $fio->{stats}{errorCnt}++;
716 if ( $attr->{size} < 16 * 1024 * 1024 ) {
718 # Cache the entire old file if it is less than 16MB
721 $fio->{rxInData} = "";
722 while ( $fh->read(\$data, 16 * 1024 * 1024) > 0 ) {
723 $fio->{rxInData} .= $data;
725 $fio->log("$attr->{fullPath}: cached all $attr->{size}"
727 if ( $fio->{logLevel} >= 9 );
730 # Create and write a temporary output file
732 unlink("$fio->{outDirSh}RStmp")
733 if ( -f "$fio->{outDirSh}RStmp" );
734 if ( open(F, "+>", "$fio->{outDirSh}RStmp") ) {
737 while ( $fh->read(\$data, 1024 * 1024) > 0 ) {
738 if ( syswrite(F, $data) != length($data) ) {
739 $fio->log(sprintf("Can't write len=%d to %s",
740 length($data) , "$fio->{outDirSh}RStmp"));
742 $fio->{stats}{errorCnt}++;
745 $byteCnt += length($data);
748 $fio->{rxInName} = "$fio->{outDirSh}RStmp";
749 sysseek($fio->{rxInFd}, 0, 0);
750 $fio->log("$attr->{fullPath}: copied $byteCnt,"
751 . "$attr->{size} bytes to $fio->{rxInName}")
752 if ( $fio->{logLevel} >= 9 );
754 $fio->log("Unable to open $fio->{outDirSh}RStmp");
756 $fio->{stats}{errorCnt}++;
762 if ( open(F, "<", $attr->{fullPath}) ) {
764 $fio->{rxInName} = $attr->{fullPath};
766 $fio->log("Unable to open $attr->{fullPath}");
767 $fio->{stats}{errorCnt}++;
772 my $lastBlk = $fio->{rxMatchNext} - 1;
773 $fio->log("$fio->{rxFile}{name}: writing blocks $fio->{rxMatchBlk}.."
775 if ( $fio->{logLevel} >= 9 );
776 my $seekPosn = $fio->{rxMatchBlk} * $fio->{rxBlkSize};
777 if ( defined($fio->{rxInFd})
778 && !sysseek($fio->{rxInFd}, $seekPosn, 0) ) {
779 $fio->log("Unable to seek $attr->{rxInName} to $seekPosn");
780 $fio->{stats}{errorCnt}++;
783 my $cnt = $fio->{rxMatchNext} - $fio->{rxMatchBlk};
784 my($thisCnt, $len, $data);
785 for ( my $i = 0 ; $i < $cnt ; $i += $thisCnt ) {
786 $thisCnt = $cnt - $i;
787 $thisCnt = 512 if ( $thisCnt > 512 );
788 if ( $fio->{rxMatchBlk} + $i + $thisCnt == $fio->{rxBlkCnt} ) {
789 $len = ($thisCnt - 1) * $fio->{rxBlkSize} + $fio->{rxRemainder};
791 $len = $thisCnt * $fio->{rxBlkSize};
793 if ( defined($fio->{rxInData}) ) {
794 $data = substr($fio->{rxInData}, $seekPosn, $len);
797 my $got = sysread($fio->{rxInFd}, $data, $len);
798 if ( $got != $len ) {
799 my $inFileSize = -s $fio->{rxInName};
800 $fio->log("Unable to read $len bytes from $fio->{rxInName}"
801 . " got=$got, seekPosn=$seekPosn"
802 . " ($i,$thisCnt,$fio->{rxBlkCnt},$inFileSize"
803 . ",$attr->{size})");
804 $fio->{stats}{errorCnt}++;
809 $fio->{rxOutFd}->write(\$data);
810 $fio->{rxDigest}->add($data);
811 $fio->{rxSize} += length($data);
813 $fio->{rxMatchBlk} = undef;
815 if ( defined($blk) ) {
817 # Remember the new block number
819 $fio->{rxMatchBlk} = $blk;
820 $fio->{rxMatchNext} = $blk + 1;
822 if ( defined($newData) ) {
824 # Write the new chunk
826 my $len = length($newData);
827 $fio->log("$fio->{rxFile}{name}: writing $len bytes new data")
828 if ( $fio->{logLevel} >= 9 );
829 $fio->{rxOutFd}->write(\$newData);
830 $fio->{rxDigest}->add($newData);
831 $fio->{rxSize} += length($newData);
836 # Finish up the current receive file. Returns undef if ok, -1 if not.
837 # Returns 1 if the md4 digest doesn't match.
842 my $name = $1 if ( $fio->{rxFile}{name} =~ /(.*)/ );
844 if ( !defined($fio->{rxDigest}) ) {
846 # File was exact match, but we still need to verify the
847 # MD4 checksum. Therefore open and read the file.
849 $fio->{rxDigest} = File::RsyncP::Digest->new;
850 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
851 my $attr = $fio->{rxLocalAttr};
852 if ( defined($attr) ) {
853 if ( defined(my $fh = BackupPC::FileZIO->open(
856 $attr->{compress})) ) {
858 while ( $fh->read(\$data, 4 * 65536) > 0 ) {
859 $fio->{rxDigest}->add($data);
860 $fio->{rxSize} += length($data);
864 $fio->log("Can't open $attr->{fullPath} for MD4 check ($name)");
865 $fio->{stats}{errorCnt}++;
868 $fio->log("$name got exact match")
869 if ( $fio->{logLevel} >= 5 );
871 close($fio->{rxInFd}) if ( defined($fio->{rxInFd}) );
872 unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" );
873 my $newDigest = $fio->{rxDigest}->digest;
874 if ( $fio->{logLevel} >= 3 ) {
875 my $md4Str = unpack("H*", $md4);
876 my $newStr = unpack("H*", $newDigest);
877 $fio->log("$name got digests $md4Str vs $newStr")
879 if ( $md4 ne $newDigest ) {
880 $fio->log("$name: fatal error: md4 doesn't match");
881 $fio->{stats}{errorCnt}++;
882 if ( defined($fio->{rxOutFd}) ) {
883 $fio->{rxOutFd}->close;
884 unlink($fio->{rxOutFile});
889 # One special case is an empty file: if the file size is
890 # zero we need to open the output file to create it.
892 if ( $fio->{rxSize} == 0 ) {
893 my $rxOutFileRel = "$fio->{shareM}/"
894 . $fio->{bpc}->fileNameMangle($name);
895 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
896 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
897 $rxOutFile, $fio->{rxSize},
898 $fio->{xfer}{compress});
900 if ( !defined($fio->{rxOutFd}) ) {
902 # No output file, meaning original was an exact match.
904 $fio->log("$name: nothing to do")
905 if ( $fio->{logLevel} >= 5 );
906 my $attr = $fio->{rxLocalAttr};
907 my $f = $fio->{rxFile};
908 $fio->logFileAction("same", $f) if ( $fio->{logLevel} >= 1 );
910 || $attr->{type} != $f->{type}
911 || $attr->{mtime} != $f->{mtime}
912 || $attr->{size} != $f->{size}
913 || $attr->{gid} != $f->{gid}
914 || $attr->{mode} != $f->{mode} ) {
916 # In the full case, or if the attributes are different,
917 # we need to make a link from the previous file and
918 # set the attributes.
920 my $rxOutFile = $fio->{outDirSh}
921 . $fio->{bpc}->fileNameMangle($name);
922 if ( !link($attr->{fullPath}, $rxOutFile) ) {
923 $fio->log("Unable to link $attr->{fullPath} to $rxOutFile");
924 $fio->{stats}{errorCnt}++;
930 $fio->{stats}{TotalFileCnt}++;
931 $fio->{stats}{TotalFileSize} += $fio->{rxSize};
932 $fio->{stats}{ExistFileCnt}++;
933 $fio->{stats}{ExistFileSize} += $fio->{rxSize};
934 $fio->{stats}{ExistFileCompSize} += -s $rxOutFile;
935 $fio->{rxFile}{size} = $fio->{rxSize};
936 return $fio->attribSet($fio->{rxFile});
939 if ( defined($fio->{rxOutFd}) ) {
940 my $exist = $fio->processClose($fio->{rxOutFd},
941 $fio->{rxOutFileRel},
943 $fio->logFileAction($exist ? "pool" : "create", $fio->{rxFile})
944 if ( $fio->{logLevel} >= 1 );
945 $fio->{rxFile}{size} = $fio->{rxSize};
946 return $fio->attribSet($fio->{rxFile});
948 delete($fio->{rxDigest});
949 delete($fio->{rxInData});
954 # Callback function for BackupPC::View->find. Note the order of the
955 # first two arguments.
959 my($a, $fio, $fList, $outputFunc) = @_;
960 my $name = $a->{relPath};
962 my $type = $fio->mode2type($a->{mode});
963 my $extraAttribs = {};
965 $n =~ s/^\Q$fio->{xfer}{pathHdrSrc}//;
966 $fio->log("Sending $name (remote=$n)") if ( $fio->{logLevel} >= 4 );
967 if ( $type == BPC_FTYPE_CHARDEV
968 || $type == BPC_FTYPE_BLOCKDEV
969 || $type == BPC_FTYPE_SYMLINK ) {
970 my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0, $a->{compress});
972 if ( defined($fh) ) {
973 if ( $fh->read(\$str, $a->{size} + 1) == $a->{size} ) {
974 if ( $type == BPC_FTYPE_SYMLINK ) {
976 # Reconstruct symbolic link
978 $extraAttribs = { link => $str };
979 } elsif ( $str =~ /(\d*),(\d*)/ ) {
981 # Reconstruct char or block special major/minor device num
983 $extraAttribs = { rdev => $1 * 256 + $2 };
985 $fio->log("$name: unexpected special file contents $str");
986 $fio->{stats}{errorCnt}++;
990 $fio->log("$name: can't read exactly $a->{size} bytes");
991 $fio->{stats}{errorCnt}++;
996 $fio->log("$name: can't open");
997 $fio->{stats}{errorCnt}++;
1002 #dev => 0, # later, when we support hardlinks
1003 #inode => 0, # later, when we support hardlinks
1007 mtime => $a->{mtime},
1012 $f->{name} = "$fio->{xfer}{pathHdrDest}/$f->{name}";
1013 $f->{name} =~ s{//+}{/}g;
1014 $fio->logFileAction("restore", $f) if ( $fio->{logLevel} >= 1 );
1015 &$outputFunc($fList->encodeData);
1019 if ( $type != BPC_FTYPE_DIR ) {
1020 $fio->{stats}{TotalFileCnt}++;
1021 $fio->{stats}{TotalFileSize} += $a->{size};
1027 my($fio, $flist, $outputFunc) = @_;
1030 # Populate the file list with the files requested by the user.
1031 # Since some might be directories so we call BackupPC::View::find.
1033 $fio->log("fileListSend: sending file list: "
1034 . join(" ", @{$fio->{fileList}})) if ( $fio->{logLevel} >= 4 );
1035 foreach my $name ( @{$fio->{fileList}} ) {
1036 $fio->{view}->find($fio->{xfer}{bkupSrcNum},
1037 $fio->{xfer}{bkupSrcShare},
1039 \&fileListEltSend, $fio, $flist, $outputFunc);
1045 my($fio, $isChild) = @_;
1048 # Flush the attributes if this is the child
1050 $fio->attribWrite(undef);
1056 # join('',@_), kill 0;