X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=bin%2FBackupPC_tarExtract;h=954da702b06691bdce5da48ba92dac5544d49980;hp=d8f63116ea156f9b18a7fa629d275a6445173011;hb=488bb662f6d144d42376b3d14e9b1e438e00e6f8;hpb=c6cebe03e53dcc49f889cf15ccda5b0fe3acd235 diff --git a/bin/BackupPC_tarExtract b/bin/BackupPC_tarExtract index d8f6311..954da70 100755 --- a/bin/BackupPC_tarExtract +++ b/bin/BackupPC_tarExtract @@ -1,4 +1,4 @@ -#!/bin/perl -T +#!/usr/bin/perl #============================================================= -*-perl-*- # # BackupPC_tarExtract: extract data from a dump @@ -9,7 +9,7 @@ # Craig Barratt # # COPYRIGHT -# Copyright (C) 2001 Craig Barratt +# Copyright (C) 2001-2009 Craig Barratt # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,45 +27,71 @@ # #======================================================================== # -# Version 2.0.0beta0, released 23 Feb 2003. +# Version 3.2.0, released 31 Jul 2010. # # See http://backuppc.sourceforge.net. # #======================================================================== use strict; +no utf8; use lib "/usr/local/BackupPC/lib"; +use Encode qw/from_to/; use BackupPC::Lib; use BackupPC::Attrib qw(:all); use BackupPC::FileZIO; use BackupPC::PoolWrite; use File::Path; +use constant S_IFMT => 0170000; # type of file + die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); my $TopDir = $bpc->TopDir(); my $BinDir = $bpc->BinDir(); my %Conf = $bpc->Conf(); if ( @ARGV != 3 ) { - print("usage: $0 \n"); + print("usage: $0 \n"); exit(1); } if ( $ARGV[0] !~ /^([\w\.\s-]+)$/ ) { - print("$0: bad host name '$ARGV[0]'\n"); + print("$0: bad client name '$ARGV[0]'\n"); exit(1); } -my $host = $1; -if ( $ARGV[1] !~ /^([\w\s\.\/\$-]+)$/ ) { +my $client = $1; +if ( $ARGV[1] =~ m{(^|/)\.\.(/|$)} ) { print("$0: bad share name '$ARGV[1]'\n"); exit(1); } -my $ShareNameUM = $1; +my $ShareNameUM = $1 if ( $ARGV[1] =~ /(.*)/ ); my $ShareName = $bpc->fileNameEltMangle($ShareNameUM); if ( $ARGV[2] !~ /^(\d+)$/ ) { print("$0: bad compress level '$ARGV[2]'\n"); exit(1); } my $Compress = $1; +my $Abort = 0; +my $AbortReason; + +# +# Re-read config file, so we can include the PC-specific config +# +if ( defined(my $error = $bpc->ConfigRead($client)) ) { + print("BackupPC_tarExtract: Can't read PC's config file: $error\n"); + exit(1); +} +%Conf = $bpc->Conf(); + +# +# Catch various signals +# +$SIG{INT} = \&catch_signal; +$SIG{ALRM} = \&catch_signal; +$SIG{TERM} = \&catch_signal; +$SIG{PIPE} = \&catch_signal; +$SIG{STOP} = \&catch_signal; +$SIG{TSTP} = \&catch_signal; +$SIG{TTIN} = \&catch_signal; # # This constant and the line of code below that uses it is borrowed @@ -76,13 +102,13 @@ my $Compress = $1; # Copyright 1998 Stephen Zander. All rights reserved. # my $tar_unpack_header - = 'Z100 A8 A8 A8 A12 A12 A8 A1 Z100 A6 A2 Z32 Z32 A8 A8 A155 x12'; + = 'Z100 A8 A8 A8 a12 A12 A8 A1 Z100 A6 A2 Z32 Z32 A8 A8 A155 x12'; my $tar_header_length = 512; my $BufSize = 1048576; # 1MB or 2^20 my $MaxFiles = 20; my $Errors = 0; -my $OutDir = "$TopDir/pc/$host/new"; +my $OutDir = "$TopDir/pc/$client/new"; my %Attrib = (); my $ExistFileCnt = 0; @@ -90,6 +116,7 @@ my $ExistFileSize = 0; my $ExistFileCompSize = 0; my $TotalFileCnt = 0; my $TotalFileSize = 0; +my $TarReadHdrCnt = 0; sub TarRead { @@ -98,12 +125,16 @@ sub TarRead $data = "\0" x $totBytes; while ( $numBytes < $totBytes ) { + return if ( $Abort ); $newBytes = sysread($fh, substr($data, $numBytes, $totBytes - $numBytes), $totBytes - $numBytes); if ( $newBytes <= 0 ) { - print(STDERR "Unexpected end of tar archive (tot = $totBytes," + return if ( $TarReadHdrCnt == 1 ); # empty tar file ok + print("Unexpected end of tar archive (tot = $totBytes," . " num = $numBytes, posn = " . sysseek($fh, 0, 1) . ")\n"); + $Abort = 1; + $AbortReason = "Unexpected end of tar archive"; $Errors++; return; } @@ -116,6 +147,7 @@ sub TarReadHeader { my($fh) = @_; + $TarReadHdrCnt++; return $1 if ( TarRead($fh, $tar_header_length) =~ /(.*)/s ); return; } @@ -139,7 +171,8 @@ sub TarReadFileInfo while ( 1 ) { $head = TarReadHeader($fh); - return if ( $head eq "" || $head eq "\0" x $tar_header_length ); + return if ( $Abort || $head eq "" + || $head eq "\0" x $tar_header_length ); ($name, # string $mode, # octal number $uid, # octal number @@ -160,9 +193,35 @@ sub TarReadFileInfo $mode = oct $mode; $uid = oct $uid; $gid = oct $gid; - $size =~ s/^6/2/; # fix bug in smbclient for >=2GB files - $size =~ s/^7/3/; # fix bug in smbclient for >=2GB files - $size = oct $size; + if ( ord($size) == 128 ) { + # + # GNU tar extension: for >=8GB files the size is stored + # in big endian binary. + # + $size = 65536 * 65536 * unpack("N", substr($size, 4, 4)) + + unpack("N", substr($size, 8, 4)); + } else { + # + # We used to have a patch here for smbclient 2.2.x. For file + # sizes between 2 and 4GB it sent the wrong size. But since + # samba 3.0.0 has been released we no longer support this + # patch since valid files could have sizes that start with + # 6 or 7 in octal (eg: 6-8GB files). + # + # $size =~ s/^6/2/; # fix bug in smbclient for >=2GB files + # $size =~ s/^7/3/; # fix bug in smbclient for >=2GB files + # + # To avoid integer overflow in case we are in the 4GB - 8GB + # range, we do the conversion in two parts. + # + if ( $size =~ /([0-9]{9,})/ ) { + my $len = length($1); + $size = oct(substr($1, 0, $len - 8)) * (1 << 24) + + oct(substr($1, $len - 8)); + } else { + $size = oct($size); + } + } $mtime = oct $mtime; $chksum = oct $chksum; $devmajor = oct $devmajor; @@ -171,7 +230,7 @@ sub TarReadFileInfo $prefix = ""; substr ($head, 148, 8) = " "; if (unpack ("%16C*", $head) != $chksum) { - print(STDERR "$name: checksum error at " + print("$name: checksum error at " . sysseek($fh, 0, 1) , "\n"); $Errors++; } @@ -188,10 +247,23 @@ sub TarReadFileInfo TarFlush($fh, $size); next; } + printf("Got file '%s', mode 0%o, size %g, type %d\n", + $name, $mode, $size, $type) if ( $Conf{XferLogLevel} >= 3 ); $name = $longName if ( defined($longName) ); $linkname = $longLink if ( defined($longLink) ); + + # + # Map client charset encodings to utf8 + # + # printf("File $name (hex: %s)\n", unpack("H*", $name)); + if ( $Conf{ClientCharset} ne "" ) { + from_to($name, $Conf{ClientCharset}, "utf8"); + from_to($linkname, $Conf{ClientCharset}, "utf8"); + } + # printf("File now $name (hex: %s)\n", unpack("H*", $name)); + $name =~ s{^\./+}{}; - $name =~ s{/+$}{}; + $name =~ s{/+\.?$}{}; $name =~ s{//+}{/}g; return { name => $name, @@ -235,7 +307,7 @@ sub TarReadFile $Attrib{$dir} = BackupPC::Attrib->new({ compress => $Compress }); if ( -f $Attrib{$dir}->fileName("$OutDir/$dir") && !$Attrib{$dir}->read("$OutDir/$dir") ) { - printf(STDERR "Unable to read attribute file %s\n", + printf("Unable to read attribute file %s\n", $Attrib{$dir}->fileName("$OutDir/$dir")); $Errors++; } @@ -244,6 +316,7 @@ sub TarReadFile # # Directory # + logFileAction("create", $f) if ( $Conf{XferLogLevel} >= 1 ); mkpath("$OutDir/$ShareName/$f->{mangleName}", 0, 0777) if ( !-d "$OutDir/$ShareName/$f->{mangleName}" ); } elsif ( $f->{type} == BPC_FTYPE_FILE ) { @@ -252,6 +325,7 @@ sub TarReadFile # my($nRead); #print("Reading $f->{name}, $f->{size} bytes, type $f->{type}\n"); + pathCreate($dir, "$OutDir/$ShareName/$f->{mangleName}", $f); my $poolWrite = BackupPC::PoolWrite->new($bpc, "$OutDir/$ShareName/$f->{mangleName}", $f->{size}, $Compress); @@ -260,14 +334,24 @@ sub TarReadFile ? $f->{size} - $nRead : $BufSize; my $data = TarRead($fh, $thisRead); if ( $data eq "" ) { - print(STDERR "Unexpected end of tar archive during read\n"); - $Errors++; + if ( !$Abort ) { + print("Unexpected end of tar archive during read\n"); + $AbortReason = "Unexpected end of tar archive"; + $Errors++; + } + $poolWrite->abort; + $Abort = 1; + unlink("$OutDir/$ShareName/$f->{mangleName}"); + print("Removing partial file $f->{name}\n"); return; } $poolWrite->write(\$data); $nRead += $thisRead; } - processClose($poolWrite, "$ShareName/$f->{mangleName}", $f->{size}); + my $exist = processClose($poolWrite, "$ShareName/$f->{mangleName}", + $f->{size}); + logFileAction($exist ? "pool" : "create", $f) + if ( $Conf{XferLogLevel} >= 1 ); TarFlush($fh, $f->{size}); } elsif ( $f->{type} == BPC_FTYPE_HARDLINK ) { # @@ -279,11 +363,15 @@ sub TarReadFile # a plain file. # $f->{size} = length($f->{linkname}); + pathCreate($dir, "$OutDir/$ShareName/$f->{mangleName}", $f); my $poolWrite = BackupPC::PoolWrite->new($bpc, "$OutDir/$ShareName/$f->{mangleName}", $f->{size}, $Compress); $poolWrite->write(\$f->{linkname}); - processClose($poolWrite, "$ShareName/$f->{mangleName}", $f->{size}); + my $exist = processClose($poolWrite, "$ShareName/$f->{mangleName}", + $f->{size}); + logFileAction($exist ? "pool" : "create", $f) + if ( $Conf{XferLogLevel} >= 1 ); } elsif ( $f->{type} == BPC_FTYPE_SYMLINK ) { # # Symbolic link: write the value of the link to a plain file, @@ -293,11 +381,15 @@ sub TarReadFile # contents. # $f->{size} = length($f->{linkname}); + pathCreate($dir, "$OutDir/$ShareName/$f->{mangleName}", $f); my $poolWrite = BackupPC::PoolWrite->new($bpc, "$OutDir/$ShareName/$f->{mangleName}", $f->{size}, $Compress); $poolWrite->write(\$f->{linkname}); - processClose($poolWrite, "$ShareName/$f->{mangleName}", $f->{size}); + my $exist = processClose($poolWrite, "$ShareName/$f->{mangleName}", + $f->{size}); + logFileAction($exist ? "pool" : "create", $f) + if ( $Conf{XferLogLevel} >= 1 ); } elsif ( $f->{type} == BPC_FTYPE_CHARDEV || $f->{type} == BPC_FTYPE_BLOCKDEV || $f->{type} == BPC_FTYPE_FIFO ) { @@ -313,12 +405,16 @@ sub TarReadFile } else { $data = "$f->{devmajor},$f->{devminor}"; } + pathCreate($dir, "$OutDir/$ShareName/$f->{mangleName}", $f); my $poolWrite = BackupPC::PoolWrite->new($bpc, "$OutDir/$ShareName/$f->{mangleName}", length($data), $Compress); $poolWrite->write(\$data); $f->{size} = length($data); - processClose($poolWrite, "$ShareName/$f->{mangleName}", length($data)); + my $exist = processClose($poolWrite, "$ShareName/$f->{mangleName}", + length($data)); + logFileAction($exist ? "pool" : "create", $f) + if ( $Conf{XferLogLevel} >= 1 ); } else { print("Got unknown type $f->{type} for $f->{name}\n"); $Errors++; @@ -346,37 +442,120 @@ sub attributeWrite my $poolWrite = BackupPC::PoolWrite->new($bpc, $fileName, length($data), $Compress); $poolWrite->write(\$data); - processClose($poolWrite, $Attrib{$d}->fileName($d), length($data)); + processClose($poolWrite, $Attrib{$d}->fileName($d), length($data), 1); } delete($Attrib{$d}); } sub processClose { - my($poolWrite, $fileName, $origSize) = @_; + my($poolWrite, $fileName, $origSize, $noStats) = @_; my($exists, $digest, $outSize, $errs) = $poolWrite->close; if ( @$errs ) { - print(STDERR join("", @$errs)); + print(join("", @$errs)); $Errors += @$errs; } - $TotalFileCnt++; - $TotalFileSize += $origSize; + if ( !$noStats ) { + $TotalFileCnt++; + $TotalFileSize += $origSize; + } if ( $exists ) { - $ExistFileCnt++; - $ExistFileSize += $origSize; - $ExistFileCompSize += $outSize; + if ( !$noStats ) { + $ExistFileCnt++; + $ExistFileSize += $origSize; + $ExistFileCompSize += $outSize; + } } elsif ( $outSize > 0 ) { print(NEW_FILES "$digest $origSize $fileName\n"); } + return $exists && $origSize > 0; +} + +# +# Generate a log file message for a completed file +# +sub logFileAction +{ + my($action, $f) = @_; + my $owner = "$f->{uid}/$f->{gid}"; + my $name = $f->{name}; + $name = "." if ( $name eq "" ); + my $type = (("", "p", "c", "", "d", "", "b", "", "", "", "l", "", "s")) + [($f->{mode} & S_IFMT) >> 12]; + $type = "h" if ( $f->{type} == BPC_FTYPE_HARDLINK ); + + printf(" %-6s %1s%4o %9s %11.0f %s\n", + $action, + $type, + $f->{mode} & 07777, + $owner, + $f->{size}, + $name); +} + +# +# Create the parent directory of $file if necessary +# +sub pathCreate +{ + my($dir, $fullPath, $f) = @_; + + # + # Get parent directory of each of $dir and $fullPath + # + # print("pathCreate: dir = $dir, fullPath = $fullPath\n"); + $dir =~ s{/([^/]*)$}{}; + my $file = $bpc->fileNameUnmangle($1); + $fullPath =~ s{/[^/]*$}{}; + return if ( -d $fullPath || $file eq "" ); + unlink($fullPath) if ( -e $fullPath ); + mkpath($fullPath, 0, 0777); + $Attrib{$dir} = BackupPC::Attrib->new({ compress => $Compress }) + if ( !defined($Attrib{$dir}) ); + # print("pathCreate: adding file = $file to dir = $dir\n"); + $Attrib{$dir}->set($file, { + type => BPC_FTYPE_DIR, + mode => 0755, + uid => $f->{uid}, + gid => $f->{gid}, + size => 0, + mtime => 0, + }); +} + +sub catch_signal +{ + my $sigName = shift; + + # + # The first time we receive a signal we try to gracefully + # abort the backup. This allows us to keep a partial dump + # with the in-progress file deleted and attribute caches + # flushed to disk etc. + # + print("BackupPC_tarExtract: got signal $sigName\n"); + if ( !$Abort ) { + $Abort++; + $AbortReason = "received signal $sigName"; + return; + } + + # + # This is a second signal: time to clean up. + # + print("BackupPC_tarExtract: quitting on second signal $sigName\n"); + close(NEW_FILES); + exit(1) } mkpath("$OutDir/$ShareName", 0, 0777); -open(NEW_FILES, ">>", "$TopDir/pc/$host/NewFileList") - || die("can't open $TopDir/pc/$host/NewFileList"); +open(NEW_FILES, ">>", "$TopDir/pc/$client/NewFileList") + || die("can't open $TopDir/pc/$client/NewFileList"); +binmode(NEW_FILES); binmode(STDIN); -1 while ( TarReadFile(*STDIN) ); -1 while ( sysread(STDIN, my $discard, 1024) ); +1 while ( !$Abort && TarReadFile(*STDIN) ); +1 while ( !$Abort && sysread(STDIN, my $discard, 1024) ); # # Flush out remaining attributes. @@ -386,6 +565,10 @@ foreach my $d ( keys(%Attrib) ) { } close(NEW_FILES); +if ( $Abort ) { + print("BackupPC_tarExtact aborting ($AbortReason)\n"); +} + # # Report results to BackupPC_dump #