1 #============================================================= -*-perl-*-
8 # Craig Barratt <cbarratt@users.sourceforge.net>
11 # Copyright (C) 2002 Craig Barratt
13 #========================================================================
15 # Version 2.0.0_CVS, released 18 Jan 2003.
17 # See http://backuppc.sourceforge.net.
19 #========================================================================
21 package BackupPC::Xfer::RsyncFileIO;
25 use BackupPC::Attrib qw(:all);
27 use BackupPC::PoolWrite;
28 use BackupPC::PoolWrite;
31 use constant S_IFMT => 0170000; # type of file
32 use constant S_IFDIR => 0040000; # directory
33 use constant S_IFCHR => 0020000; # character special
34 use constant S_IFBLK => 0060000; # block special
35 use constant S_IFREG => 0100000; # regular
36 use constant S_IFLNK => 0120000; # symbolic link
37 use constant S_IFSOCK => 0140000; # socket
38 use constant S_IFIFO => 0010000; # fifo
40 use vars qw( $RsyncLibOK );
43 eval "use File::RsyncP::Digest";
46 # Rsync module doesn't exist.
56 my($class, $options) = @_;
58 return if ( !$RsyncLibOK );
63 digest => File::RsyncP::Digest->new,
66 logHandler => \&logHandler,
72 ExistFileCompSize => 0,
77 $fio->{shareM} = $fio->{bpc}->fileNameEltMangle($fio->{share});
78 $fio->{outDir} = "$fio->{xfer}{outDir}/new/";
79 $fio->{outDirSh} = "$fio->{outDir}/$fio->{shareM}/";
80 $fio->{view} = BackupPC::View->new($fio->{bpc}, $fio->{client},
82 $fio->{full} = $fio->{xfer}{type} eq "full" ? 1 : 0;
83 $fio->{newFilesFH} = $fio->{xfer}{newFilesFH};
89 my($fio, $value) = @_;
91 $fio->{blockSize} = $value if ( defined($value) );
92 return $fio->{blockSize};
98 $fio->{logHandler} = $sub;
102 # Setup rsync checksum computation for the given file.
106 my($fio, $f, $needMD4) = @_;
108 my $attr = $fio->attribGet($f);
110 $fio->csumEnd if ( defined($fio->{fh}) );
111 return if ( $attr->{type} != BPC_FTYPE_FILE );
112 if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath},
114 $attr->{compress})) ) {
115 $fio->log("Can't open $attr->{fullPath} (name=$f->{name})");
119 $fio->{csumDigest} = File::RsyncP::Digest->new;
120 $fio->{csumDigest}->add(pack("V", $fio->{checksumSeed}));
122 delete($fio->{csumDigest});
124 alarm($fio->{timeout}) if ( defined($fio->{timeout}) );
129 my($fio, $num, $csumLen, $blockSize) = @_;
135 return if ( !defined($fio->{fh}) );
136 if ( $fio->{fh}->read(\$fileData, $blockSize * $num) <= 0 ) {
139 $fio->{csumDigest}->add($fileData) if ( defined($fio->{csumDigest}) );
140 $fio->log(sprintf("%s: getting csum ($num,$csumLen,%d,0x%x)\n",
143 $fio->{checksumSeed}))
144 if ( $fio->{logLevel} >= 10 );
145 return $fio->{digest}->blockDigest($fileData, $blockSize,
146 $csumLen, $fio->{checksumSeed});
153 return if ( !defined($fio->{fh}) );
155 # make sure we read the entire file for the file MD4 digest
157 if ( defined($fio->{csumDigest}) ) {
159 while ( $fio->{fh}->read(\$fileData, 65536) > 0 ) {
160 $fio->{csumDigest}->add($fileData);
165 return $fio->{csumDigest}->digest if ( defined($fio->{csumDigest}) );
172 my $attr = $fio->attribGet($f);
174 $fio->readEnd if ( defined($fio->{fh}) );
175 if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath},
177 $attr->{compress})) ) {
178 $fio->log("Can't open $attr->{fullPath} (name=$f->{name})");
181 $fio->log("$f->{name}: opened for read") if ( $fio->{logLevel} >= 4 );
182 alarm($fio->{timeout}) if ( defined($fio->{timeout}) );
191 return if ( !defined($fio->{fh}) );
192 if ( $fio->{fh}->read(\$fileData, $num) <= 0 ) {
193 return $fio->readEnd;
195 $fio->log(sprintf("read returns %d bytes", length($fileData)))
196 if ( $fio->{logLevel} >= 8 );
204 return if ( !defined($fio->{fh}) );
206 $fio->log("closing $fio->{file}{name})") if ( $fio->{logLevel} >= 8 );
213 my($fio, $checksumSeed) = @_;
215 $fio->{checksumSeed} = $checksumSeed;
220 my($fio, $localDir, $remoteDir) = @_;
222 $fio->{localDir} = $localDir;
223 $fio->{remoteDir} = $remoteDir;
228 my($fio, $share, $dir) = @_;
231 #$fio->log("viewCacheDir($share, $dir)");
232 if ( !defined($share) ) {
233 $share = $fio->{share};
234 $shareM = $fio->{shareM};
236 $shareM = $fio->{bpc}->fileNameEltMangle($share);
238 $shareM = "$shareM/$dir" if ( $dir ne "" );
239 return if ( defined($fio->{viewCache}{$shareM}) );
241 # purge old cache entries (ie: those that don't match the
242 # first part of $dir).
244 foreach my $d ( keys(%{$fio->{viewCache}}) ) {
245 delete($fio->{viewCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
248 # fetch new directory attributes
250 $fio->{viewCache}{$shareM}
251 = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir);
257 my($dir, $fname, $share, $shareM);
260 $fname = "$fio->{xfer}{pathHdrSrc}/$fname"
261 if ( defined($fio->{xfer}{pathHdrSrc}) );
262 $fname =~ s{//+}{/}g;
263 if ( $fname =~ m{(.*)/(.*)} ) {
264 $shareM = $fio->{shareM};
267 } elsif ( $fname ne "." ) {
268 $shareM = $fio->{shareM};
274 $fname = $fio->{share};
276 $fio->viewCacheDir($share, $dir);
277 $shareM .= "/$dir" if ( $dir ne "" );
278 return $fio->{viewCache}{$shareM}{$fname};
283 my($fio, $mode) = @_;
285 if ( ($mode & S_IFMT) == S_IFREG ) {
286 return BPC_FTYPE_FILE;
287 } elsif ( ($mode & S_IFMT) == S_IFDIR ) {
288 return BPC_FTYPE_DIR;
289 } elsif ( ($mode & S_IFMT) == S_IFLNK ) {
290 return BPC_FTYPE_SYMLINK;
291 } elsif ( ($mode & S_IFMT) == S_IFCHR ) {
292 return BPC_FTYPE_CHARDEV;
293 } elsif ( ($mode & S_IFMT) == S_IFBLK ) {
294 return BPC_FTYPE_BLOCKDEV;
295 } elsif ( ($mode & S_IFMT) == S_IFIFO ) {
296 return BPC_FTYPE_FIFO;
297 } elsif ( ($mode & S_IFMT) == S_IFSOCK ) {
298 return BPC_FTYPE_SOCKET;
300 return BPC_FTYPE_UNKNOWN;
305 # Set the attributes for a file. Returns non-zero on error.
309 my($fio, $f, $placeHolder) = @_;
312 if ( $f->{name} =~ m{(.*)/(.*)} ) {
314 $dir = "$fio->{shareM}/" . $1;
315 } elsif ( $f->{name} eq "." ) {
317 $file = $fio->{share};
319 $dir = $fio->{shareM};
323 if ( !defined($fio->{attribLastDir}) || $fio->{attribLastDir} ne $dir ) {
325 # Flush any directories that don't match the first part
326 # of the new directory
328 foreach my $d ( keys(%{$fio->{attrib}}) ) {
329 next if ( $d eq "" || "$dir/" =~ m{^\Q$d/} );
330 $fio->attribWrite($d);
332 $fio->{attribLastDir} = $dir;
334 if ( !exists($fio->{attrib}{$dir}) ) {
335 $fio->{attrib}{$dir} = BackupPC::Attrib->new({
336 compress => $fio->{xfer}{compress},
338 my $path = $fio->{outDir} . $dir;
339 if ( -f $fio->{attrib}{$dir}->fileName($path)
340 && !$fio->{attrib}{$dir}->read($path) ) {
341 $fio->log(sprintf("Unable to read attribute file %s",
342 $fio->{attrib}{$dir}->fileName($path)));
345 $fio->log("attribSet(dir=$dir, file=$file)") if ( $fio->{logLevel} >= 4 );
347 $fio->{attrib}{$dir}->set($file, {
348 type => $fio->mode2type($f->{mode}),
352 size => $placeHolder ? -1 : $f->{size},
353 mtime => $f->{mtime},
363 if ( !defined($d) ) {
365 # flush all entries (in reverse order)
367 foreach $d ( sort({$b cmp $a} keys(%{$fio->{attrib}})) ) {
368 $fio->attribWrite($d);
372 return if ( !defined($fio->{attrib}{$d}) );
374 # Set deleted files in the attributes. Any file in the view
375 # that doesn't have attributes is deleted. All files sent by
376 # rsync have attributes temporarily set so we can do deletion
377 # detection. We also prune these temporary attributes.
383 $dir = $1 if ( $d =~ m{.+?/(.*)} );
384 $fio->viewCacheDir(undef, $dir);
385 ##print("attribWrite $d,$dir\n");
386 ##$Data::Dumper::Indent = 1;
387 ##$fio->log("attribWrite $d,$dir");
388 ##$fio->log("viewCacheLogKeys = ", keys(%{$fio->{viewCache}}));
389 ##$fio->log("attribKeys = ", keys(%{$fio->{attrib}}));
390 ##print "viewCache = ", Dumper($fio->{attrib});
391 ##print "attrib = ", Dumper($fio->{attrib});
392 if ( defined($fio->{viewCache}{$d}) ) {
393 foreach my $f ( keys(%{$fio->{viewCache}{$d}}) ) {
395 $name = "$1/$name" if ( $d =~ m{.*?/(.*)} );
396 if ( defined(my $a = $fio->{attrib}{$d}->get($f)) ) {
398 # delete temporary attributes (skipped files)
400 if ( $a->{size} < 0 ) {
401 $fio->{attrib}{$d}->set($f, undef);
402 $fio->logFileAction("skip", {
403 %{$fio->{viewCache}{$d}{$f}},
405 }) if ( $fio->{logLevel} >= 2 );
408 ##print("Delete file $f\n");
409 $fio->logFileAction("delete", {
410 %{$fio->{viewCache}{$d}{$f}},
412 }) if ( $fio->{logLevel} >= 1 );
413 $fio->{attrib}{$d}->set($f, {
414 type => BPC_FTYPE_DELETED,
425 if ( $fio->{attrib}{$d}->fileCount ) {
426 my $data = $fio->{attrib}{$d}->writeData;
429 $dirM = $1 . "/" . $fio->{bpc}->fileNameMangle($2)
430 if ( $dirM =~ m{(.*?)/(.*)} );
431 my $fileName = $fio->{attrib}{$d}->fileName("$fio->{outDir}$dirM");
432 $fio->log("attribWrite(dir=$d) -> $fileName")
433 if ( $fio->{logLevel} >= 4 );
434 my $poolWrite = BackupPC::PoolWrite->new($fio->{bpc}, $fileName,
435 length($data), $fio->{xfer}{compress});
436 $poolWrite->write(\$data);
437 $fio->processClose($poolWrite, $fio->{attrib}{$d}->fileName($d),
440 delete($fio->{attrib}{$d});
445 my($fio, $poolWrite, $fileName, $origSize, $doStats) = @_;
446 my($exists, $digest, $outSize, $errs) = $poolWrite->close;
448 $fileName =~ s{^/+}{};
449 $fio->log(@$errs) if ( defined($errs) && @$errs );
451 $fio->{stats}{TotalFileCnt}++;
452 $fio->{stats}{TotalFileSize} += $origSize;
456 $fio->{stats}{ExistFileCnt}++;
457 $fio->{stats}{ExistFileSize} += $origSize;
458 $fio->{stats}{ExistFileCompSize} += $outSize;
460 } elsif ( $outSize > 0 ) {
461 my $fh = $fio->{newFilesFH};
462 print($fh "$digest $origSize $fileName\n") if ( defined($fh) );
464 return $exists && $origSize > 0;
471 return $fio->{stats};
475 # Make a given directory. Returns non-zero on error.
480 my $name = $1 if ( $f->{name} =~ /(.*)/ );
483 if ( $name eq "." ) {
484 $path = $fio->{outDirSh};
486 $path = $fio->{outDirSh} . $fio->{bpc}->fileNameMangle($name);
488 $fio->logFileAction("create", $f) if ( $fio->{logLevel} >= 1 );
489 $fio->log("makePath($path, 0777)") if ( $fio->{logLevel} >= 5 );
490 $path = $1 if ( $path =~ /(.*)/ );
491 File::Path::mkpath($path, 0, 0777) if ( !-d $path );
492 return $fio->attribSet($f) if ( -d $path );
493 $fio->log("Can't create directory $path");
498 # Make a special file. Returns non-zero on error.
503 my $name = $1 if ( $f->{name} =~ /(.*)/ );
504 my $fNameM = $fio->{bpc}->fileNameMangle($name);
505 my $path = $fio->{outDirSh} . $fNameM;
506 my $attr = $fio->attribGet($f);
508 my $type = $fio->mode2type($f->{mode});
510 $fio->log("makeSpecial($path, $type, $f->{mode})")
511 if ( $fio->{logLevel} >= 5 );
512 if ( $type == BPC_FTYPE_CHARDEV || $type == BPC_FTYPE_BLOCKDEV ) {
513 my($major, $minor, $fh, $fileData);
515 $major = $f->{rdev} >> 8;
516 $minor = $f->{rdev} & 0xff;
517 $str = "$major,$minor";
518 } elsif ( ($f->{mode} & S_IFMT) == S_IFLNK ) {
522 # Now see if the file is different, or this is a full, in which
523 # case we create the new file.
528 || $attr->{type} != $fio->mode2type($f->{mode})
529 || $attr->{mtime} != $f->{mtime}
530 || $attr->{size} != $f->{size}
531 || $attr->{uid} != $f->{uid}
532 || $attr->{gid} != $f->{gid}
533 || $attr->{mode} != $f->{mode}
534 || !defined($fh = BackupPC::FileZIO->open($attr->{fullPath}, 0,
536 || $fh->read(\$fileData, length($str) + 1) != length($str)
537 || $fileData ne $str ) {
538 $fh->close if ( defined($fh) );
539 $fh = BackupPC::PoolWrite->new($fio->{bpc}, $path,
540 length($str), $fio->{xfer}{compress});
542 my $exist = $fio->processClose($fh, "$fio->{shareM}/$fNameM",
544 $fio->logFileAction($exist ? "pool" : "create", $f)
545 if ( $fio->{logLevel} >= 1 );
546 return $fio->attribSet($f);
548 $fio->logFileAction("skip", $f) if ( $fio->{logLevel} >= 2 );
550 $fh->close if ( defined($fh) );
555 my($fio, $path) = @_;
557 $fio->log("Unexpected call BackupPC::Xfer::RsyncFileIO->unlink($path)");
561 # Default log handler
567 print(STDERR $str, "\n");
571 # Handle one or more log messages
575 my($fio, @logStr) = @_;
577 foreach my $str ( @logStr ) {
578 next if ( $str eq "" );
579 $fio->{logHandler}($str);
584 # Generate a log file message for a completed file
588 my($fio, $action, $f) = @_;
589 my $owner = "$f->{uid}/$f->{gid}";
590 my $type = (("", "p", "c", "", "d", "", "b", "", "", "", "l", "", "s"))
591 [($f->{mode} & S_IFMT) >> 12];
593 $fio->log(sprintf(" %-6s %1s%4o %9s %11.0f %s",
603 # Later we'll use this function to complete a prior unfinished dump.
604 # We'll do an incremental on the part we have already, and then a
605 # full or incremental against the rest.
613 # Start receive of file deltas for a particular file.
617 my($fio, $f, $cnt, $size, $remainder) = @_;
619 $fio->{rxFile} = $f; # remote file attributes
620 $fio->{rxLocalAttr} = $fio->attribGet($f); # local file attributes
621 $fio->{rxBlkCnt} = $cnt; # how many blocks we will receive
622 $fio->{rxBlkSize} = $size; # block size
623 $fio->{rxRemainder} = $remainder; # size of the last block
624 $fio->{rxMatchBlk} = 0; # current start of match
625 $fio->{rxMatchNext} = 0; # current next block of match
626 my $rxSize = ($cnt - 1) * $size + $remainder;
627 if ( $fio->{rxFile}{size} != $rxSize ) {
628 $fio->{rxMatchBlk} = undef; # size different, so no file match
629 $fio->log("$fio->{rxFile}{name}: size doesn't match"
630 . " ($fio->{rxFile}{size} vs $rxSize)")
631 if ( $fio->{logLevel} >= 5 );
633 delete($fio->{rxInFd});
634 delete($fio->{rxOutFd});
635 delete($fio->{rxDigest});
636 delete($fio->{rxInData});
637 alarm($fio->{timeout}) if ( defined($fio->{timeout}) );
641 # Process the next file delta for the current file. Returns 0 if ok,
642 # -1 if not. Must be called with either a block number, $blk, or new data,
643 # $newData, (not both) defined.
647 my($fio, $blk, $newData) = @_;
649 if ( defined($blk) ) {
650 if ( defined($fio->{rxMatchBlk}) && $fio->{rxMatchNext} == $blk ) {
652 # got the next block in order; just keep track.
654 $fio->{rxMatchNext}++;
658 my $newDataLen = length($newData);
659 $fio->log("$fio->{rxFile}{name}: blk=$blk, newData=$newDataLen, rxMatchBlk=$fio->{rxMatchBlk}, rxMatchNext=$fio->{rxMatchNext}")
660 if ( $fio->{logLevel} >= 8 );
661 if ( !defined($fio->{rxOutFd}) ) {
663 # maybe the file has no changes
665 if ( $fio->{rxMatchNext} == $fio->{rxBlkCnt}
666 && !defined($blk) && !defined($newData) ) {
667 #$fio->log("$fio->{rxFile}{name}: file is unchanged");
668 # if ( $fio->{logLevel} >= 8 );
673 # need to open an output file where we will build the
676 $fio->{rxFile}{name} =~ /(.*)/;
677 my $rxOutFileRel = "$fio->{shareM}/" . $fio->{bpc}->fileNameMangle($1);
678 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
679 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
680 $rxOutFile, $fio->{rxFile}{size},
681 $fio->{xfer}{compress});
682 $fio->log("$fio->{rxFile}{name}: opening output file $rxOutFile")
683 if ( $fio->{logLevel} >= 10 );
684 $fio->{rxOutFile} = $rxOutFile;
685 $fio->{rxOutFileRel} = $rxOutFileRel;
686 $fio->{rxDigest} = File::RsyncP::Digest->new;
687 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
689 if ( defined($fio->{rxMatchBlk})
690 && $fio->{rxMatchBlk} != $fio->{rxMatchNext} ) {
692 # Need to copy the sequence of blocks that matched. If the file
693 # is compressed we need to make a copy of the uncompressed file,
694 # since the compressed file is not seekable. Future optimizations
695 # would be to keep the uncompressed file in memory (eg, up to say
696 # 10MB), only create an uncompressed copy if the matching
697 # blocks were not monotonic, and to only do this if there are
698 # matching blocks (eg, maybe the entire file is new).
700 my $attr = $fio->{rxLocalAttr};
702 if ( !defined($fio->{rxInFd}) && !defined($fio->{rxInData}) ) {
703 if ( $attr->{compress} ) {
704 if ( !defined($fh = BackupPC::FileZIO->open(
707 $attr->{compress})) ) {
708 $fio->log("Can't open $attr->{fullPath}");
711 if ( $attr->{size} < 10 * 1024 * 1024 ) {
713 # Cache the entire old file if it is less than 10MB
716 $fio->{rxInData} = "";
717 while ( $fh->read(\$data, 10 * 1024 * 1024) > 0 ) {
718 $fio->{rxInData} .= $data;
722 # Create and write a temporary output file
724 unlink("$fio->{outDirSh}RStmp")
725 if ( -f "$fio->{outDirSh}RStmp" );
726 if ( open(F, ">+", "$fio->{outDirSh}RStmp") ) {
728 while ( $fh->read(\$data, 1024 * 1024) > 0 ) {
729 if ( syswrite(F, $data) != length($data) ) {
730 $fio->log(sprintf("Can't write len=%d to %s",
731 length($data) , "$fio->{outDirSh}RStmp"));
737 $fio->{rxInName} = "$fio->{outDirSh}RStmp";
738 seek($fio->{rxInFd}, 0, 0);
740 $fio->log("Unable to open $fio->{outDirSh}RStmp");
747 if ( open(F, "<", $attr->{fullPath}) ) {
749 $fio->{rxInName} = $attr->{fullPath};
751 $fio->log("Unable to open $attr->{fullPath}");
756 my $lastBlk = $fio->{rxMatchNext} - 1;
757 $fio->log("$fio->{rxFile}{name}: writing blocks $fio->{rxMatchBlk}.."
759 if ( $fio->{logLevel} >= 10 );
760 my $seekPosn = $fio->{rxMatchBlk} * $fio->{rxBlkSize};
761 if ( defined($fio->{rxInFd}) && !seek($fio->{rxInFd}, $seekPosn, 0) ) {
762 $fio->log("Unable to seek $attr->{fullPath} to $seekPosn");
765 my $cnt = $fio->{rxMatchNext} - $fio->{rxMatchBlk};
766 my($thisCnt, $len, $data);
767 for ( my $i = 0 ; $i < $cnt ; $i += $thisCnt ) {
768 $thisCnt = $cnt - $i;
769 $thisCnt = 512 if ( $thisCnt > 512 );
770 if ( $fio->{rxMatchBlk} + $i + $thisCnt == $fio->{rxBlkCnt} ) {
771 $len = ($thisCnt - 1) * $fio->{rxBlkSize} + $fio->{rxRemainder};
773 $len = $thisCnt * $fio->{rxBlkSize};
775 if ( defined($fio->{rxInData}) ) {
776 $data = substr($fio->{rxInData}, $seekPosn, $len);
778 if ( sysread($fio->{rxInFd}, $data, $len) != $len ) {
779 $fio->log("Unable to read $len bytes from"
780 . " $fio->{rxInName} "
781 . "($i,$thisCnt,$fio->{rxBlkCnt})");
785 $fio->{rxOutFd}->write(\$data);
786 $fio->{rxDigest}->add($data);
788 $fio->{rxMatchBlk} = undef;
790 if ( defined($blk) ) {
792 # Remember the new block number
794 $fio->{rxMatchBlk} = $blk;
795 $fio->{rxMatchNext} = $blk + 1;
797 if ( defined($newData) ) {
799 # Write the new chunk
801 my $len = length($newData);
802 $fio->log("$fio->{rxFile}{name}: writing $len bytes new data")
803 if ( $fio->{logLevel} >= 10 );
804 $fio->{rxOutFd}->write(\$newData);
805 $fio->{rxDigest}->add($newData);
810 # Finish up the current receive file. Returns undef if ok, -1 if not.
811 # Returns 1 if the md4 digest doesn't match.
816 my $name = $1 if ( $fio->{rxFile}{name} =~ /(.*)/ );
818 if ( !defined($fio->{rxDigest}) ) {
820 # File was exact match, but we still need to verify the
821 # MD4 checksum. Therefore open and read the file.
823 $fio->{rxDigest} = File::RsyncP::Digest->new;
824 $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
825 my $attr = $fio->{rxLocalAttr};
826 if ( defined($attr) && defined(my $fh = BackupPC::FileZIO->open(
829 $attr->{compress})) ) {
831 while ( $fh->read(\$data, 4 * 65536) > 0 ) {
832 $fio->{rxDigest}->add($data);
838 $fio->log("$name got exact match")
839 if ( $fio->{logLevel} >= 5 );
841 close($fio->{rxInFd}) if ( defined($fio->{rxInFd}) );
842 unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" );
843 my $newDigest = $fio->{rxDigest}->digest;
844 if ( $fio->{logLevel} >= 3 ) {
845 my $md4Str = unpack("H*", $md4);
846 my $newStr = unpack("H*", $newDigest);
847 $fio->log("$name got digests $md4Str vs $newStr")
849 if ( $md4 ne $newDigest ) {
850 $fio->log("$name md4 doesn't match")
851 if ( $fio->{logLevel} >= 1 );
852 if ( defined($fio->{rxOutFd}) ) {
853 $fio->{rxOutFd}->close;
854 unlink($fio->{rxOutFile});
859 # One special case is an empty file: if the file size is
860 # zero we need to open the output file to create it.
862 if ( $fio->{rxFile}{size} == 0 ) {
863 my $rxOutFileRel = "$fio->{shareM}/"
864 . $fio->{bpc}->fileNameMangle($name);
865 my $rxOutFile = $fio->{outDir} . $rxOutFileRel;
866 $fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc},
867 $rxOutFile, $fio->{rxFile}{size},
868 $fio->{xfer}{compress});
870 if ( !defined($fio->{rxOutFd}) ) {
872 # No output file, meaning original was an exact match.
874 $fio->log("$name: nothing to do")
875 if ( $fio->{logLevel} >= 5 );
876 my $attr = $fio->{rxLocalAttr};
877 my $f = $fio->{rxFile};
878 $fio->logFileAction("same", $f) if ( $fio->{logLevel} >= 1 );
880 || $attr->{type} != $f->{type}
881 || $attr->{mtime} != $f->{mtime}
882 || $attr->{size} != $f->{size}
883 || $attr->{gid} != $f->{gid}
884 || $attr->{mode} != $f->{mode} ) {
886 # In the full case, or if the attributes are different,
887 # we need to make a link from the previous file and
888 # set the attributes.
890 my $rxOutFile = $fio->{outDirSh}
891 . $fio->{bpc}->fileNameMangle($name);
892 if ( !link($attr->{fullPath}, $rxOutFile) ) {
893 $fio->log("Unable to link $attr->{fullPath} to $rxOutFile");
899 $fio->{stats}{TotalFileCnt}++;
900 $fio->{stats}{TotalFileSize} += $fio->{rxFile}{size};
901 $fio->{stats}{ExistFileCnt}++;
902 $fio->{stats}{ExistFileSize} += $fio->{rxFile}{size};
903 $fio->{stats}{ExistFileCompSize} += -s $rxOutFile;
907 if ( defined($fio->{rxOutFd}) ) {
908 my $exist = $fio->processClose($fio->{rxOutFd},
909 $fio->{rxOutFileRel},
910 $fio->{rxFile}{size}, 1);
911 $fio->logFileAction($exist ? "pool" : "create", $fio->{rxFile})
912 if ( $fio->{logLevel} >= 1 );
914 delete($fio->{rxDigest});
915 delete($fio->{rxInData});
920 # Callback function for BackupPC::View->find. Note the order of the
921 # first two arguments.
925 my($a, $fio, $fList, $outputFunc) = @_;
926 my $name = $a->{relPath};
928 my $type = $fio->mode2type($a->{mode});
929 my $extraAttribs = {};
931 $n =~ s/^\Q$fio->{xfer}{pathHdrSrc}//;
932 $fio->log("Sending $name (remote=$n)") if ( $fio->{logLevel} >= 4 );
933 if ( $type == BPC_FTYPE_CHARDEV
934 || $type == BPC_FTYPE_BLOCKDEV
935 || $type == BPC_FTYPE_SYMLINK ) {
936 my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0, $a->{compress});
938 if ( defined($fh) ) {
939 if ( $fh->read(\$str, $a->{size} + 1) == $a->{size} ) {
940 if ( $type == BPC_FTYPE_SYMLINK ) {
942 # Reconstruct symbolic link
944 $extraAttribs = { link => $str };
945 } elsif ( $str =~ /(\d*),(\d*)/ ) {
947 # Reconstruct char or block special major/minor device num
949 $extraAttribs = { rdev => $1 * 256 + $2 };
952 $fio->log("$name: unexpected file contents $str");
956 $fio->log("$name: can't read exactly $a->{size} bytes");
961 $fio->log("$name: can't open");
966 #dev => 0, # later, when we support hardlinks
967 #inode => 0, # later, when we support hardlinks
971 mtime => $a->{mtime},
976 $f->{name} = "$fio->{xfer}{pathHdrDest}/$f->{name}";
977 $f->{name} =~ s{//+}{/}g;
978 $fio->logFileAction("restore", $f) if ( $fio->{logLevel} >= 1 );
979 &$outputFunc($fList->encodeData);
983 if ( $type != BPC_FTYPE_DIR ) {
984 $fio->{stats}{TotalFileCnt}++;
985 $fio->{stats}{TotalFileSize} += $a->{size};
987 alarm($fio->{timeout}) if ( defined($fio->{timeout}) );
992 my($fio, $flist, $outputFunc) = @_;
995 # Populate the file list with the files requested by the user.
996 # Since some might be directories so we call BackupPC::View::find.
998 $fio->log("fileListSend: sending file list: "
999 . join(" ", @{$fio->{fileList}})) if ( $fio->{logLevel} >= 4 );
1000 foreach my $name ( @{$fio->{fileList}} ) {
1001 $fio->{view}->find($fio->{xfer}{bkupSrcNum},
1002 $fio->{xfer}{bkupSrcShare},
1004 \&fileListEltSend, $fio, $flist, $outputFunc);
1010 my($fio, $isChild) = @_;
1013 # Flush the attributes if this is the child
1015 $fio->attribWrite(undef);
1016 alarm($fio->{timeout}) if ( defined($fio->{timeout}) );
1022 # join('',@_), kill 0;