X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=bin%2FBackupPC_tarCreate;h=b04464a41de1cb34c90bc544e3868879a94b0871;hp=844ab9746d46b5fc8ada76a65fef478a8267bfa1;hb=bc72241a845cd61a6386ec1394ca9d49c3fae3de;hpb=1ce7d1541ea1279aaa0a75c16986a3fd40b608ec diff --git a/bin/BackupPC_tarCreate b/bin/BackupPC_tarCreate index 844ab97..b04464a 100755 --- a/bin/BackupPC_tarCreate +++ b/bin/BackupPC_tarCreate @@ -1,4 +1,4 @@ -#!/bin/perl -T +#!/usr/bin/perl #============================================================= -*-perl-*- # # BackupPC_tarCreate: create a tar archive of an existing dump @@ -6,20 +6,27 @@ # # DESCRIPTION # -# Usage: BackupPC_tarCreate [-t] [-h host] [-n dumpNum] [-s shareName] -# [-r pathRemove] [-p pathAdd] files/directories... +# Usage: BackupPC_tarCreate [options] files/directories... # # Flags: # Required options: # -# -h host host from which the tar archive is created -# -n dumpNum dump number from which the tar archive is created -# -s shareName share name from which the tar archive is created +# -h host Host from which the tar archive is created. +# -n dumpNum Dump number from which the tar archive is created. +# A negative number means relative to the end (eg -1 +# means the most recent dump, -2 2nd most recent etc). +# -s shareName Share name from which the tar archive is created. # # Other options: # -t print summary totals # -r pathRemove path prefix that will be replaced with pathAdd # -p pathAdd new path prefix +# -b BLOCKS output write buffer size in 512-byte blocks (default 20; same as tar) +# -w readBufSz buffer size for reading files (default 1048576 = 1MB) +# -e charset charset for encoding file names (default: value of +# $Conf{ClientCharset} when backup was done) +# -l just print a file listing; don't generate an archive +# -L just print a detailed file listing; don't generate an archive # # The -h, -n and -s options specify which dump is used to generate # the tar archive. The -r and -p options can be used to relocate @@ -30,7 +37,7 @@ # Craig Barratt # # COPYRIGHT -# Copyright (C) 2001 Craig Barratt +# Copyright (C) 2001-2009 Craig Barratt # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -48,89 +55,103 @@ # #======================================================================== # -# Version 1.5.0, released 2 Aug 2002. +# Version 3.2.0, released 31 Jul 2010. # # See http://backuppc.sourceforge.net. # #======================================================================== use strict; -use lib "__INSTALLDIR__/lib"; +no utf8; +use lib "/usr/local/BackupPC/lib"; use File::Path; use Getopt::Std; +use Encode qw/from_to/; use BackupPC::Lib; use BackupPC::Attrib qw(:all); use BackupPC::FileZIO; +use BackupPC::View; die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); -my $TopDir = $bpc->TopDir(); -my $BinDir = $bpc->BinDir(); -my %Conf = $bpc->Conf(); my %opts; -getopts("th:n:p:r:s:", \%opts); -if ( @ARGV < 1 ) { - print(STDERR "usage: $0 [-t] [-h host] [-n dumpNum] [-s shareName]" - . " [-r pathRemove] [-p pathAdd]" - . " files/directories...\n"); +if ( !getopts("Llte:h:n:p:r:s:b:w:i", \%opts) || @ARGV < 1 ) { + print STDERR <BackupInfoRead($Host); -my($Compress, $Mangle, $CompressF, $MangleF, $NumF, $i); my $FileCnt = 0; my $ByteCnt = 0; my $DirCnt = 0; my $SpecialCnt = 0; my $ErrorCnt = 0; +my $i; +$Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 ); for ( $i = 0 ; $i < @Backups ; $i++ ) { - if ( !$Backups[$i]{noFill} ) { - # - # Remember the most recent filled backup - # - $NumF = $Backups[$i]{num}; - $MangleF = $Backups[$i]{mangle}; - $CompressF = $Backups[$i]{compress}; - } - next if ( $Backups[$i]{num} != $Num ); - $Compress = $Backups[$i]{compress}; - $Mangle = $Backups[$i]{mangle}; - if ( !$Backups[$i]{noFill} ) { - # no need to back-fill a filled backup - $NumF = $MangleF = $CompressF = undef; - } - last; + last if ( $Backups[$i]{num} == $Num ); } if ( $i >= @Backups ) { print(STDERR "$0: bad backup number $Num for host $Host\n"); exit(1); } +my $Charset = $Backups[$i]{charset}; +$Charset = $opts{e} if ( $opts{e} ne "" ); + my $PathRemove = $1 if ( $opts{r} =~ /(.+)/ ); my $PathAdd = $1 if ( $opts{p} =~ /(.+)/ ); -if ( $opts{s} !~ /^([\w\s\.\/\$-]+)$/ ) { +if ( $opts{s} =~ m{(^|/)\.\.(/|$)} ) { print(STDERR "$0: bad share name '$opts{s}'\n"); exit(1); } -my $ShareNameOrig = $opts{s}; -my $ShareName = $Mangle ? $bpc->fileNameEltMangle($ShareNameOrig) - : $ShareNameOrig; -my $ShareNameF = $MangleF ? $bpc->fileNameEltMangle($ShareNameOrig) - : $ShareNameOrig; + +# XXX ASA Search extension +my $view_opts; + +my %Conf = $bpc->Conf; +if ( $Conf{TarCreateIncremental} || $opts{i} ) { + warn "# incremental dump"; + $view_opts = { only_first => 1 }; +} + +our $ShareName = $opts{s}; +our $view = BackupPC::View->new($bpc, $Host, \@Backups, $view_opts); # # This constant and the line of code below that uses it are borrowed @@ -144,9 +165,9 @@ my $tar_pack_header = 'a100 a8 a8 a8 a12 a12 A8 a1 a100 a6 a2 a32 a32 a8 a8 a155 x12'; my $tar_header_length = 512; -my $BufSize = 1048576; # 1MB or 2^20 +my $BufSize = $opts{w} || 1048576; # 1MB or 2^20 my $WriteBuf = ""; -my $WriteBufSz = 20 * $tar_header_length; +my $WriteBufSz = ($opts{b} || 20) * $tar_header_length; my(%UidCache, %GidCache); my(%HardLinkExtraFiles, @HardLinks); @@ -154,31 +175,36 @@ my(%HardLinkExtraFiles, @HardLinks); # # Write out all the requested files/directories # +binmode(STDOUT); my $fh = *STDOUT; -foreach my $dir ( @ARGV ) { - archiveWrite($fh, $dir); -} - -# -# Write out any hardlinks (if any) -# -foreach my $hdr ( @HardLinks ) { - $hdr->{size} = 0; - if ( defined($PathRemove) - && substr($hdr->{linkname}, 0, length($PathRemove)+1) - eq ".$PathRemove" ) { - substr($hdr->{linkname}, 0, length($PathRemove)+1) = ".$PathAdd"; +if ( $ShareName eq "*" ) { + my $PathRemoveOrig = $PathRemove; + my $PathAddOrig = $PathAdd; + foreach $ShareName ( $view->shareList($Num) ) { + #print(STDERR "Doing share ($ShareName)\n"); + $PathRemove = "/" if ( !defined($PathRemoveOrig) ); + ($PathAdd = "/$ShareName/$PathAddOrig") =~ s{//+}{/}g; + foreach my $dir ( @ARGV ) { + archiveWrite($fh, $dir); + } + archiveWriteHardLinks($fh); + } +} else { + foreach my $dir ( @ARGV ) { + archiveWrite($fh, $dir); } - TarWriteFileInfo($fh, $hdr); + archiveWriteHardLinks($fh); } -# -# Finish with two null 512 byte headers, and then round out a full -# block. -# -my $data = "\0" x ($tar_header_length * 2); -TarWrite($fh, \$data); -TarWrite($fh, undef); +if ( !$opts{l} && !$opts{L} ) { + # + # Finish with two null 512 byte headers, and then round out a full + # block. + # + my $data = "\0" x ($tar_header_length * 2); + TarWrite($fh, \$data); + TarWrite($fh, undef); +} # # print out totals if requested @@ -187,6 +213,13 @@ if ( $opts{t} ) { print STDERR "Done: $FileCnt files, $ByteCnt bytes, $DirCnt dirs,", " $SpecialCnt specials, $ErrorCnt errors\n"; } +if ( $ErrorCnt && !$FileCnt && !$DirCnt ) { + # + # Got errors, with no files or directories; exit with non-zero + # status + # + exit(1); +} exit(0); ########################################################################### @@ -196,37 +229,44 @@ exit(0); sub archiveWrite { my($fh, $dir, $tarPathOverride) = @_; - if ( $dir =~ m{(^|/)\.\.(/|$)} || $dir !~ /^(.*)$/ ) { + + if ( $dir =~ m{(^|/)\.\.(/|$)} ) { print(STDERR "$0: bad directory '$dir'\n"); $ErrorCnt++; - next; - } - (my $DirOrig = $1) =~ s{/+$}{}; - $DirOrig =~ s{^\.?/+}{}; - my($Dir, $DirF, $FullPath, $FullPathF); - if ( $DirOrig eq "" ) { - $Dir = $DirF = ""; - $FullPath = "$TopDir/pc/$Host/$Num/$ShareName"; - $FullPathF = "$TopDir/pc/$Host/$NumF/$ShareNameF" - if ( defined($NumF) ); - } else { - $Dir = $Mangle ? $bpc->fileNameMangle($DirOrig) : $DirOrig; - $DirF = $MangleF ? $bpc->fileNameMangle($DirOrig) : $DirOrig; - $FullPath = "$TopDir/pc/$Host/$Num/$ShareName/$Dir"; - $FullPathF = "$TopDir/pc/$Host/$NumF/$ShareNameF/$DirF" - if ( defined($NumF) ); + return; } - if ( -f $FullPath ) { - TarWriteFile($fh, $FullPath, $Mangle, $Compress, $tarPathOverride); - } elsif ( -d $FullPath || (defined($NumF) && -d $FullPathF) ) { - MergeFind($fh, $FullPath, $FullPathF); - } elsif ( defined($NumF) && -f $FullPathF ) { - TarWriteFile($fh, $FullPathF, $MangleF, $CompressF, $tarPathOverride); - } else { - print(STDERR "$0: $Host, backup $Num, doesn't have a directory or file" - . " $ShareNameOrig/$DirOrig\n"); + $dir = "/" if ( $dir eq "." ); + #print(STDERR "calling find with $Num, $ShareName, $dir\n"); + if ( $view->find($Num, $ShareName, $dir, 0, \&TarWriteFile, + $fh, $tarPathOverride) < 0 ) { + print(STDERR "$0: bad share or directory '$ShareName/$dir'\n"); $ErrorCnt++; + return; + } +} + +# +# Write out any hardlinks (if any) +# +sub archiveWriteHardLinks +{ + my($fh) = @_; + foreach my $hdr ( @HardLinks ) { + $hdr->{size} = 0; + my $name = $hdr->{linkname}; + $name =~ s{^\./}{/}; + if ( defined($HardLinkExtraFiles{$name}) ) { + $hdr->{linkname} = $HardLinkExtraFiles{$name}; + } + if ( defined($PathRemove) + && substr($hdr->{linkname}, 0, length($PathRemove)+1) + eq ".$PathRemove" ) { + substr($hdr->{linkname}, 0, length($PathRemove)+1) = ".$PathAdd"; + } + TarWriteFileInfo($fh, $hdr); } + @HardLinks = (); + %HardLinkExtraFiles = (); } sub UidLookup @@ -266,13 +306,13 @@ sub TarWrite my $done = $WriteBufSz - length($WriteBuf); if ( syswrite($fh, $WriteBuf . substr($$dataRef, 0, $done)) != $WriteBufSz ) { - print(STDERR "Unable to write to output file\n"); + print(STDERR "Unable to write to output file ($!)\n"); exit(1); } while ( $done + $WriteBufSz <= length($$dataRef) ) { if ( syswrite($fh, substr($$dataRef, $done, $WriteBufSz)) != $WriteBufSz ) { - print(STDERR "Unable to write to output file\n"); + print(STDERR "Unable to write to output file ($!)\n"); exit(1); } $done += $WriteBufSz; @@ -300,12 +340,29 @@ sub TarWriteHeader : ""; my $devminor = defined($hdr->{devminor}) ? sprintf("%07o", $hdr->{devminor}) : ""; + my $sizeStr; + if ( $hdr->{size} >= 2 * 65536 * 65536 ) { + # + # GNU extension for files >= 8GB: send size in big-endian binary + # + $sizeStr = pack("c4 N N", 0x80, 0, 0, 0, + $hdr->{size} / (65536 * 65536), + $hdr->{size} % (65536 * 65536)); + } elsif ( $hdr->{size} >= 1 * 65536 * 65536 ) { + # + # sprintf octal only handles up to 2^32 - 1 + # + $sizeStr = sprintf("%03o", $hdr->{size} / (1 << 24)) + . sprintf("%08o", $hdr->{size} % (1 << 24)); + } else { + $sizeStr = sprintf("%011o", $hdr->{size}); + } my $data = pack($tar_pack_header, substr($hdr->{name}, 0, 99), sprintf("%07o", $hdr->{mode}), sprintf("%07o", $hdr->{uid}), sprintf("%07o", $hdr->{gid}), - sprintf("%011o", $hdr->{size}), + $sizeStr, sprintf("%011o", $hdr->{mtime}), "", #checksum field - space padded by pack("A8") $hdr->{type}, @@ -326,6 +383,36 @@ sub TarWriteFileInfo { my($fh, $hdr) = @_; + # + # Convert path names to requested (eg: client) charset + # + if ( $Charset ne "" ) { + from_to($hdr->{name}, "utf8", $Charset); + from_to($hdr->{linkname}, "utf8", $Charset); + } + + if ( $opts{l} ) { + print($hdr->{name} . "\n"); + return; + } elsif ( $opts{L} ) { + my $owner = "$hdr->{uid}/$hdr->{gid}"; + + my $name = $hdr->{name}; + + if ( $hdr->{type} == BPC_FTYPE_SYMLINK + || $hdr->{type} == BPC_FTYPE_HARDLINK ) { + $name .= " -> $hdr->{linkname}"; + } + $name =~ s/\n/\\n/g; + + printf("%6o %9s %11.0f %s\n", + $hdr->{mode}, + $owner, + $hdr->{size}, + $name); + return; + } + # # Handle long link names (symbolic links) # @@ -339,6 +426,7 @@ sub TarWriteFileInfo TarWrite($fh, \$data); TarWritePad($fh, length($data)); } + # # Handle long file names # @@ -360,63 +448,12 @@ my $AttrDir; sub TarWriteFile { - my($fh, $fullName, $mangle, $compress, $tarPathOverride) = @_; - my($tarPath); + my($hdr, $fh, $tarPathOverride) = @_; - if ( $fullName =~ m{^\Q$TopDir/pc/$Host/$Num/$ShareName\E(.*)} - || (defined($NumF) - && $fullName =~ m{^\Q$TopDir/pc/$Host/$NumF/$ShareNameF\E(.*)}) ) { - $tarPath = $mangle ? $bpc->fileNameUnmangle($1) : $1; - } else { - print(STDERR "Unexpected file name from find: $fullName\n"); - return; - } + my $tarPath = $hdr->{relPath}; $tarPath = $tarPathOverride if ( defined($tarPathOverride) ); - (my $dir = $fullName) =~ s{/([^/]*)$}{}; - my $fileName = $mangle ? $bpc->fileNameUnmangle($1) : $1; - if ( $mangle && $AttrDir ne $dir ) { - $AttrDir = $dir; - $Attr = BackupPC::Attrib->new({ compress => $compress }); - if ( -f $Attr->fileName($dir) && !$Attr->read($dir) ) { - print(STDERR "Can't read attribute file in $dir\n"); - $ErrorCnt++; - $Attr = undef; - } - } - my $hdr = $Attr->get($fileName) if ( defined($Attr) ); - if ( !defined($hdr) ) { - # - # No attributes. Must be an old style backup. Reconstruct - # what we can. Painful part is computing the size if compression - # is on: only method is to uncompress the file. - # - my @s = stat($fullName); - $hdr = { - type => -d _ ? BPC_FTYPE_DIR : BPC_FTYPE_FILE, - mode => $s[2], - uid => $s[4], - gid => $s[5], - size => -f _ ? $s[7] : 0, - mtime => $s[9], - }; - if ( $compress && -f _ ) { - # - # Compute the correct size by reading the whole file - # - my $f = BackupPC::FileZIO->open($fullName, 0, $compress); - if ( !defined($f) ) { - print(STDERR "Unable to open file $fullName\n"); - $ErrorCnt++; - return; - } - my($data, $size); - while ( $f->read(\$data, $BufSize) > 0 ) { - $size += length($data); - } - $f->close; - $hdr->{size} = $size; - } - } + + $tarPath =~ s{//+}{/}g; if ( defined($PathRemove) && substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) { substr($tarPath, 0, length($PathRemove)) = $PathAdd; @@ -433,23 +470,45 @@ sub TarWriteFile TarWriteFileInfo($fh, $hdr); $DirCnt++; } elsif ( $hdr->{type} == BPC_FTYPE_FILE ) { + my($data, $size); # # Regular file: write the header and file # - my $f = BackupPC::FileZIO->open($fullName, 0, $compress); + my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress}); if ( !defined($f) ) { - print(STDERR "Unable to open file $fullName\n"); + print(STDERR "Unable to open file $hdr->{fullPath}\n"); $ErrorCnt++; return; } TarWriteFileInfo($fh, $hdr); - my($data, $size); - while ( $f->read(\$data, $BufSize) > 0 ) { - TarWrite($fh, \$data); - $size += length($data); + if ( $opts{l} || $opts{L} ) { + $size = $hdr->{size}; + } else { + while ( $f->read(\$data, $BufSize) > 0 ) { + if ( $size + length($data) > $hdr->{size} ) { + print(STDERR "Error: truncating $hdr->{fullPath} to" + . " $hdr->{size} bytes\n"); + $data = substr($data, 0, $hdr->{size} - $size); + $ErrorCnt++; + } + TarWrite($fh, \$data); + $size += length($data); + } + $f->close; + if ( $size != $hdr->{size} ) { + print(STDERR "Error: padding $hdr->{fullPath} to $hdr->{size}" + . " bytes from $size bytes\n"); + $ErrorCnt++; + while ( $size < $hdr->{size} ) { + my $len = $hdr->{size} - $size; + $len = $BufSize if ( $len > $BufSize ); + $data = "\0" x $len; + TarWrite($fh, \$data); + $size += $len; + } + } + TarWritePad($fh, $size); } - $f->close; - TarWritePad($fh, $size); $FileCnt++; $ByteCnt += $size; } elsif ( $hdr->{type} == BPC_FTYPE_HARDLINK ) { @@ -460,9 +519,9 @@ sub TarWriteFile # # Start by reading the contents of the link. # - my $f = BackupPC::FileZIO->open($fullName, 0, $compress); + my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress}); if ( !defined($f) ) { - print(STDERR "Unable to open file $fullName\n"); + print(STDERR "Unable to open file $hdr->{fullPath}\n"); $ErrorCnt++; return; } @@ -478,13 +537,14 @@ sub TarWriteFile my $done = 0; my $name = $hdr->{linkname}; $name =~ s{^\./}{/}; - if ( $HardLinkExtraFiles{$name} ) { + if ( defined($HardLinkExtraFiles{$name}) ) { $done = 1; } else { foreach my $arg ( @ARGV ) { + $arg = "/" if ( $arg eq "." ); $arg =~ s{^\./+}{/}; $arg =~ s{/+$}{}; - $done = 1 if ( $name eq $arg || $name =~ /^\Q$arg\// ); + $done = 1 if ( $name eq $arg || $name =~ /^\Q$arg\// || $arg eq "" ); } } if ( $done ) { @@ -500,17 +560,19 @@ sub TarWriteFile # routine, so that we save the hassle of dealing with # mangling, merging and attributes. # - $HardLinkExtraFiles{$hdr->{linkname}} = 1; - archiveWrite($fh, $hdr->{linkname}, $hdr->{name}); + my $name = $hdr->{linkname}; + $name =~ s{^\./}{/}; + $HardLinkExtraFiles{$name} = $hdr->{name}; + archiveWrite($fh, $name, $hdr->{name}); } } elsif ( $hdr->{type} == BPC_FTYPE_SYMLINK ) { # # Symbolic link: read the symbolic link contents into the header # and write the header. # - my $f = BackupPC::FileZIO->open($fullName, 0, $compress); + my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress}); if ( !defined($f) ) { - print(STDERR "Unable to open symlink file $fullName\n"); + print(STDERR "Unable to open symlink file $hdr->{fullPath}\n"); $ErrorCnt++; return; } @@ -530,11 +592,12 @@ sub TarWriteFile # major and minor numbers from a plain file. # if ( $hdr->{type} != BPC_FTYPE_FIFO ) { - my $f = BackupPC::FileZIO->open($fullName, 0, $compress); + my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, + $hdr->{compress}); my $data; if ( !defined($f) || $f->read(\$data, $BufSize) < 0 ) { print(STDERR "Unable to open/read char/block special file" - . " $fullName\n"); + . " $hdr->{fullPath}\n"); $f->close if ( defined($f) ); $ErrorCnt++; return; @@ -548,68 +611,14 @@ sub TarWriteFile $hdr->{size} = 0; TarWriteFileInfo($fh, $hdr); $SpecialCnt++; + } elsif ( $hdr->{type} == BPC_FTYPE_SOCKET + || $hdr->{type} == BPC_FTYPE_UNKNOWN ) { + # + # ignore these two file types - these are dynamic file types created + # by applications as needed + # } else { print(STDERR "Got unknown type $hdr->{type} for $hdr->{name}\n"); $ErrorCnt++; } } - -# -# Does a recursive find of $dir, filling in from the (filled dump) -# directory $dirF. Handles the cases where $dir and $dirF might -# or might not be mangled etc. -# -sub MergeFind -{ - my($fh, $dir, $dirF) = @_; - - my(@Dir, $fLast); - if ( -d $dir ) { - TarWriteFile($fh, $dir, $Mangle, $Compress); - } elsif ( -d $dirF ) { - TarWriteFile($fh, $dirF, $MangleF, $CompressF); - } - if ( opendir(DIR, $dir) ) { - @Dir = readdir(DIR); - closedir(DIR); - } - if ( defined($NumF) && opendir(DIR, $dirF) ) { - if ( $Mangle == $MangleF ) { - @Dir = (@Dir, readdir(DIR)); - } else { - foreach my $f ( readdir(DIR) ) { - if ( $Mangle ) { - push(@Dir, $bpc->fileNameMangle($f)); - } else { - push(@Dir, $bpc->fileNameUnmangle($f)); - } - } - } - } - foreach my $f ( sort({$a cmp $b} @Dir) ) { - next if ( $f eq "." || $f eq ".." - || $f eq $fLast || ($Mangle && $f eq "attrib") ); - $fLast = $f; - my($fF) = $f; - if ( $Mangle != $MangleF ) { - $fF = $Mangle ? $bpc->fileNameUnmangle($f) - : $bpc->fileNameMangle($f); - } - if ( -e "$dir/$f" ) { - if ( -d "$dir/$f" ) { - MergeFind($fh, "$dir/$f", "$dirF/$fF"); - } else { - TarWriteFile($fh, "$dir/$f", $Mangle, $Compress); - } - } elsif ( -e "$dirF/$fF" ) { - if ( -d "$dirF/$fF" ) { - MergeFind($fh, "$dir/$f", "$dirF/$fF"); - } else { - TarWriteFile($fh, "$dirF/$fF", $MangleF, $CompressF); - } - } else { - print(STDERR "$0: Botch on $dir, $dirF, $f, $fF\n"); - $ErrorCnt++; - } - } -}