X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=bin%2FBackupPC_dump;h=cc633f9b868b5192f1618eee4a665f5b1751c458;hp=2f3b7a336205b2ba3983add809cc2c333df7bd2e;hb=488bb662f6d144d42376b3d14e9b1e438e00e6f8;hpb=8f5625f9cdf47f8e04be654c4954838b2df3e26d diff --git a/bin/BackupPC_dump b/bin/BackupPC_dump index 2f3b7a3..cc633f9 100755 --- a/bin/BackupPC_dump +++ b/bin/BackupPC_dump @@ -1,11 +1,11 @@ -#!/bin/perl +#!/usr/bin/perl #============================================================= -*-perl-*- # # BackupPC_dump: Dump a single client. # # DESCRIPTION # -# Usage: BackupPC_dump [-i] [-f] [-d] [-e] [-v] +# Usage: BackupPC_dump [-i] [-f] [-F] [-I] [-d] [-e] [-v] # # Flags: # @@ -14,6 +14,13 @@ # # -f Do a full dump, overriding any scheduling. # +# -I Do an increment dump if the regular schedule requires a +# full or incremental, otherwise do nothing (a full is done +# if no dumps have yet succeeded) +# +# -F Do a full dump if the regular schedule requires a +# full or incremental, otherwise do nothing +# # -d Host is a DHCP pool address, and the client argument # just an IP address. We lookup the NetBios name from # the IP address. @@ -31,9 +38,9 @@ # full or incremental backup needs to be run. If no backup is # scheduled, or a ping to $client fails, then BackupPC_dump quits. # -# The backup is done using the selected XferMethod (smb, tar, rsync etc), -# extracting the dump into $TopDir/pc/$client/new. The xfer output is -# put into $TopDir/pc/$client/XferLOG. +# The backup is done using the selected XferMethod (smb, tar, rsync, +# backuppcd etc), extracting the dump into $TopDir/pc/$client/new. +# The xfer output is put into $TopDir/pc/$client/XferLOG. # # If the dump succeeds (based on parsing the output of the XferMethod): # - $TopDir/pc/$client/new is renamed to $TopDir/pc/$client/nnn, where @@ -52,7 +59,7 @@ # Craig Barratt # # COPYRIGHT -# Copyright (C) 2001-2003 Craig Barratt +# Copyright (C) 2001-2009 Craig Barratt # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -70,7 +77,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 3.2.0, released 31 Jul 2010. # # See http://backuppc.sourceforge.net. # @@ -81,9 +88,9 @@ no utf8; use lib "/usr/local/BackupPC/lib"; use BackupPC::Lib; use BackupPC::FileZIO; -use BackupPC::Xfer::Smb; -use BackupPC::Xfer::Tar; -use BackupPC::Xfer::Rsync; +use BackupPC::Storage; +use BackupPC::Xfer; +use Encode; use Socket; use File::Path; use File::Find; @@ -105,14 +112,19 @@ my $Abort; $bpc->ChildInit(); my %opts; -if ( !getopts("defiv", \%opts) || @ARGV != 1 ) { - print("usage: $0 [-d] [-e] [-f] [-i] [-v] \n"); +if ( !getopts("defivFI", \%opts) || @ARGV != 1 ) { + print("usage: $0 [-d] [-e] [-f] [-i] [-F] [-I] [-v] \n"); exit(1); } if ( $ARGV[0] !~ /^([\w\.\s-]+)$/ ) { print("$0: bad client name '$ARGV[0]'\n"); exit(1); } +if ( (defined($opts{f}) + defined($opts{i}) + defined($opts{F}) + defined($opts{I})) > 1 ) { + print("$0: exiting because you can only use one of -f, -i, -F, and -I\n"); + exit(1); +} + my $client = $1; # BackupPC's client name (might not be real host name) my $hostIP; # this is the IP address my $host; # this is the real host name @@ -158,6 +170,7 @@ if ( !defined($Hosts->{$client}) ) { my $Dir = "$TopDir/pc/$client"; my @xferPid = (); my $tarPid = -1; +my $completionPercent; # # Re-read config file, so we can include the PC-specific config @@ -191,7 +204,30 @@ mkpath($Dir, 0, 0777) if ( !-d $Dir ); if ( !-f "$Dir/LOCK" ) { open(LOCK, ">", "$Dir/LOCK") && close(LOCK); } -open(LOG, ">>", "$Dir/LOG"); + +my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); +my $logPath = sprintf("$Dir/LOG.%02d%04d", $mon + 1, $year + 1900); + +if ( !-f $logPath ) { + # + # Compress and prune old log files + # + my $lastLog = $Conf{MaxOldPerPCLogFiles} - 1; + foreach my $file ( $bpc->sortedPCLogFiles($client) ) { + if ( $lastLog <= 0 ) { + unlink($file); + next; + } + $lastLog--; + next if ( $file =~ /\.z$/ || !$Conf{CompressLevel} ); + BackupPC::FileZIO->compressCopy($file, + "$file.z", + undef, + $Conf{CompressLevel}, 1); + } +} + +open(LOG, ">>", $logPath); select(LOG); $| = 1; select(STDOUT); # @@ -211,33 +247,6 @@ if ($Conf{XferMethod} eq "archive" ) { exit(0); } -if ( !$opts{d} ) { - # - # In the non-DHCP case, make sure the host can be looked up - # via NS, or otherwise find the IP address via NetBios. - # - if ( $Conf{ClientNameAlias} ne "" ) { - $host = $Conf{ClientNameAlias}; - } else { - $host = $client; - } - if ( !defined(gethostbyname($host)) ) { - # - # Ok, NS doesn't know about it. Maybe it is a NetBios name - # instead. - # - print(STDERR "Name server doesn't know about $host; trying NetBios\n") - if ( $opts{v} ); - if ( !defined($hostIP = $bpc->NetBiosHostIPFind($host)) ) { - print(LOG $bpc->timeStamp, "Can't find host $host via netbios\n"); - print("host not found\n"); - exit(1); - } - } else { - $hostIP = $host; - } -} - ########################################################################### # Figure out what to do and do it ########################################################################### @@ -271,16 +280,27 @@ if ( $opts{d} ) { print("DHCP $hostIP $clientURI\n"); } -my($needLink, @Backups, $type, $lastBkupNum, $lastFullBkupNum); -my $lastFull = 0; -my $lastIncr = 0; +my($needLink, @Backups, $type); +my($incrBaseTime, $incrBaseBkupNum, $incrBaseLevel, $incrLevel); +my $lastFullTime = 0; +my $lastIncrTime = 0; my $partialIdx = -1; my $partialNum; +my $partialFileCnt; +my $lastBkupNum; +my $lastPartial = 0; + +# +# Maintain backward compatibility with $Conf{FullPeriod} == -1 or -2 +# meaning disable backups +# +$Conf{BackupsDisable} = -$Conf{FullPeriod} + if ( !$Conf{BackupsDisable} && $Conf{FullPeriod} < 0 ); -if ( $Conf{FullPeriod} == -1 && !$opts{f} && !$opts{i} - || $Conf{FullPeriod} == -2 ) { +if ( $Conf{BackupsDisable} == 1 && !$opts{f} && !$opts{i} + || $Conf{BackupsDisable} == 2 ) { print(STDERR "Exiting because backups are disabled with" - . " \$Conf{FullPeriod} = $Conf{FullPeriod}\n") if ( $opts{v} ); + . " \$Conf{BackupsDisable} = $Conf{BackupsDisable}\n") if ( $opts{v} ); # # Tell BackupPC to ignore old failed backups on hosts that # have backups disabled. @@ -298,28 +318,16 @@ if ( !$opts{i} && !$opts{f} && $Conf{BlackoutGoodCnt} >= 0 my($currHours) = $hour + $min / 60 + $sec / 3600; my $blackout; - # - # Handle backward compatibility with original separate scalar - # parameters. - # - if ( defined($Conf{BlackoutHourBegin}) ) { - push(@{$Conf{BlackoutPeriods}}, - { - hourBegin => $Conf{BlackoutHourBegin}, - hourEnd => $Conf{BlackoutHourEnd}, - weekDays => $Conf{BlackoutWeekDays}, - } - ); - } foreach my $p ( @{$Conf{BlackoutPeriods}} ) { # - # Allow blackout to span midnight (specified by BlackoutHourBegin - # being greater than BlackoutHourEnd) + # Allow blackout to span midnight (specified by hourBegin + # being greater than hourEnd) # next if ( ref($p->{weekDays}) ne "ARRAY" || !defined($p->{hourBegin}) || !defined($p->{hourEnd}) ); + my $matchWday = $wday; if ( $p->{hourBegin} > $p->{hourEnd} ) { $blackout = $p->{hourBegin} <= $currHours || $currHours <= $p->{hourEnd}; @@ -329,14 +337,14 @@ if ( !$opts{i} && !$opts{f} && $Conf{BlackoutGoodCnt} >= 0 # weekday check (eg: Monday 11pm-1am means Monday 2300 to # Tuesday 0100, not Monday 2300-2400 plus Monday 0000-0100). # - $wday--; - $wday += 7 if ( $wday < 0 ); + $matchWday--; + $matchWday += 7 if ( $matchWday < 0 ); } } else { $blackout = $p->{hourBegin} <= $currHours && $currHours <= $p->{hourEnd}; } - if ( $blackout && grep($_ == $wday, @{$p->{weekDays}}) ) { + if ( $blackout && grep($_ == $matchWday, @{$p->{weekDays}}) ) { # print(LOG $bpc->timeStamp, "skipping because of blackout" # . " (alive $StatusHost{aliveCnt} times)\n"); print(STDERR "Skipping $client because of blackout\n") @@ -347,7 +355,7 @@ if ( !$opts{i} && !$opts{f} && $Conf{BlackoutGoodCnt} >= 0 } if ( !$opts{i} && !$opts{f} && $StatusHost{backoffTime} > time ) { - printf(LOG "%sskipping because of user requested delay (%.1f hours left)", + printf(LOG "%sskipping because of user requested delay (%.1f hours left)\n", $bpc->timeStamp, ($StatusHost{backoffTime} - time) / 3600); NothingToDo($needLink); } @@ -357,44 +365,151 @@ if ( !$opts{i} && !$opts{f} && $StatusHost{backoffTime} > time ) { # BackupExpire($client); +my(@lastIdxByLevel, $incrCntSinceFull); + # # Read Backup information, and find times of the most recent full and -# incremental backups +# incremental backups. Also figure out which backup we will use +# as a starting point for an incremental. # @Backups = $bpc->BackupInfoRead($client); for ( my $i = 0 ; $i < @Backups ; $i++ ) { $needLink = 1 if ( $Backups[$i]{nFilesNew} eq "" || -f "$Dir/NewFileList.$Backups[$i]{num}" ); - $lastBkupNum = $Backups[$i]{num}; if ( $Backups[$i]{type} eq "full" ) { - if ( $lastFull < $Backups[$i]{startTime} ) { - $lastFull = $Backups[$i]{startTime}; - $lastFullBkupNum = $Backups[$i]{num}; + $incrCntSinceFull = 0; + $lastBkupNum = $Backups[$i]{num}; + $lastIdxByLevel[0] = $i; + if ( $lastFullTime < $Backups[$i]{startTime} ) { + $lastFullTime = $Backups[$i]{startTime}; } } elsif ( $Backups[$i]{type} eq "incr" ) { - $lastIncr = $Backups[$i]{startTime} - if ( $lastIncr < $Backups[$i]{startTime} ); + $incrCntSinceFull++; + $lastBkupNum = $Backups[$i]{num}; + $lastIdxByLevel[$Backups[$i]{level}] = $i; + $lastIncrTime = $Backups[$i]{startTime} + if ( $lastIncrTime < $Backups[$i]{startTime} ); } elsif ( $Backups[$i]{type} eq "partial" ) { - $partialIdx = $i; - $partialNum = $Backups[$i]{num}; + $partialIdx = $i; + $lastPartial = $Backups[$i]{startTime}; + $partialNum = $Backups[$i]{num}; + $partialFileCnt = $Backups[$i]{nFiles}; } } # # Decide whether we do nothing, or a full or incremental backup. # -if ( @Backups == 0 +my $needs_full = (time - $lastFullTime > $Conf{FullPeriod} * 24 * 3600 + && time - $lastIncrTime > $Conf{IncrPeriod} * 24 * 3600); +my $needs_incr = (time - $lastIncrTime > $Conf{IncrPeriod} * 24 * 3600 + && time - $lastFullTime > $Conf{IncrPeriod} * 24 * 3600); + +if ( $lastFullTime == 0 || $opts{f} - || (!$opts{i} && (time - $lastFull > $Conf{FullPeriod} * 24*3600 - && time - $lastIncr > $Conf{IncrPeriod} * 24*3600)) ) { + || (!$opts{i} && !$opts{I} && $needs_full) + || ( $opts{F} && $needs_incr) ) { $type = "full"; -} elsif ( $opts{i} || (time - $lastIncr > $Conf{IncrPeriod} * 24*3600 - && time - $lastFull > $Conf{IncrPeriod} * 24*3600) ) { + $incrLevel = 0; + $incrBaseBkupNum = $lastBkupNum; +} elsif ( $opts{i} + || $needs_incr + || ($opts{I} && $needs_full) ) { $type = "incr"; + # + # For an incremental backup, figure out which level we should + # do and the index of the reference backup, which is the most + # recent backup at any lower level. + # + @{$Conf{IncrLevels}} = [$Conf{IncrLevels}] + unless ref($Conf{IncrLevels}) eq "ARRAY"; + @{$Conf{IncrLevels}} = [1] if ( !@{$Conf{IncrLevels}} ); + $incrCntSinceFull = $incrCntSinceFull % @{$Conf{IncrLevels}}; + $incrLevel = $Conf{IncrLevels}[$incrCntSinceFull]; + for ( my $i = 0 ; $i < $incrLevel ; $i++ ) { + my $idx = $lastIdxByLevel[$i]; + next if ( !defined($idx) ); + if ( !defined($incrBaseTime) + || $Backups[$idx]{startTime} > $incrBaseTime ) { + $incrBaseBkupNum = $Backups[$idx]{num}; + $incrBaseLevel = $Backups[$idx]{level}; + $incrBaseTime = $Backups[$idx]{startTime}; + } + } + # + # Can't find any earlier lower-level backup! Shouldn't + # happen - just do full instead + # + if ( !defined($incrBaseBkupNum) || $incrLevel < 1 ) { + $type = "full"; + $incrBaseBkupNum = $lastBkupNum; + } } else { NothingToDo($needLink); } +# +# Create top-level directories if they don't exist +# +foreach my $dir ( ( + "$Conf{TopDir}", + "$Conf{TopDir}/pool", + "$Conf{TopDir}/cpool", + "$Conf{TopDir}/pc", + "$Conf{TopDir}/trash", + ) ) { + next if ( -d $dir ); + mkpath($dir, 0, 0750); + if ( !-d $dir ) { + print("Failed to create $dir\n"); + printf(LOG "%sFailed to create directory %s\n", $bpc->timeStamp, $dir); + print("link $clientURI\n") if ( $needLink ); + exit(1); + } else { + printf(LOG "%sCreated directory %s\n", $bpc->timeStamp, $dir); + } +} + +if ( !$bpc->HardlinkTest($Dir, "$TopDir/cpool") ) { + print(LOG $bpc->timeStamp, "Can't create a test hardlink between a file" + . " in $Dir and $TopDir/cpool. Either these are different" + . " file systems, or this file system doesn't support hardlinks," + . " or these directories don't exist, or there is a permissions" + . " problem, or the file system is out of inodes or full. Use" + . " df, df -i, and ls -ld to check each of these possibilities." + . " Quitting...\n"); + print("test hardlink between $Dir and $TopDir/cpool failed\n"); + print("link $clientURI\n") if ( $needLink ); + exit(1); +} + +if ( !$opts{d} ) { + # + # In the non-DHCP case, make sure the host can be looked up + # via NS, or otherwise find the IP address via NetBios. + # + if ( $Conf{ClientNameAlias} ne "" ) { + $host = $Conf{ClientNameAlias}; + } else { + $host = $client; + } + if ( !defined(gethostbyname($host)) ) { + # + # Ok, NS doesn't know about it. Maybe it is a NetBios name + # instead. + # + print(STDERR "Name server doesn't know about $host; trying NetBios\n") + if ( $opts{v} ); + if ( !defined($hostIP = $bpc->NetBiosHostIPFind($host)) ) { + print(LOG $bpc->timeStamp, "Can't find host $host via netbios\n"); + print("host not found\n"); + exit(1); + } + } else { + $hostIP = $host; + } +} + # # Check if $host is alive # @@ -432,7 +547,11 @@ $bpc->RmTreeDefer("$TopDir/trash", "$Dir/new") if ( -d "$Dir/new" ); # # Setup file extension for compression and open XferLOG output file # -$Conf{CompressLevel} = 0 if ( !BackupPC::FileZIO->compOk ); +if ( $Conf{CompressLevel} && !BackupPC::FileZIO->compOk ) { + print(LOG $bpc->timeStamp, "dump failed: can't find Compress::Zlib\n"); + print("dump failed: can't find Compress::Zlib\n"); + exit(1); +} my $fileExt = $Conf{CompressLevel} > 0 ? ".z" : ""; my $XferLOG = BackupPC::FileZIO->open("$Dir/XferLOG$fileExt", 1, $Conf{CompressLevel}); @@ -444,10 +563,10 @@ if ( !defined($XferLOG) ) { } # -# Ignore the partial dump in the case of an incremental. -# A partial is a partial full. +# Ignore the partial dump in the case of an incremental +# or when the partial is too old. A partial is a partial full. # -if ( $type ne "full" ) { +if ( $type ne "full" || time - $lastPartial > $Conf{PartialAgeMax} * 24*3600 ) { $partialNum = undef; $partialIdx = -1; } @@ -486,57 +605,65 @@ my $sizeTotal = 0; my($logMsg, %stat, $xfer, $ShareNames, $noFilesErr); my $newFilesFH; -if ( $Conf{XferMethod} eq "tar" ) { - $ShareNames = $Conf{TarShareName}; -} elsif ( $Conf{XferMethod} eq "rsync" || $Conf{XferMethod} eq "rsyncd" ) { - $ShareNames = $Conf{RsyncShareName}; -} else { - $ShareNames = $Conf{SmbShareName}; -} - -$ShareNames = [ $ShareNames ] unless ref($ShareNames) eq "ARRAY"; +$ShareNames = BackupPC::Xfer::getShareNames(\%Conf); # # Run an optional pre-dump command # UserCommandRun("DumpPreUserCmd"); +if ( $? && $Conf{UserCmdCheckStatus} ) { + print(LOG $bpc->timeStamp, + "DumpPreUserCmd returned error status $?... exiting\n"); + $XferLOG->write(\"DumpPreUserCmd returned error status $?... exiting\n"); + $stat{hostError} = "DumpPreUserCmd returned error status $?"; + BackupFailCleanup(); +} $NeedPostCmd = 1; # # Now backup each of the shares # +my $shareDuplicate = {}; for my $shareName ( @$ShareNames ) { local(*RH, *WH); + # + # Convert $shareName to utf8 octets + # + $shareName = encode("utf8", $shareName); $stat{xferOK} = $stat{hostAbort} = undef; $stat{hostError} = $stat{lastOutputLine} = undef; - if ( -d "$Dir/new/$shareName" ) { + if ( $shareName eq "" ) { + print(LOG $bpc->timeStamp, + "unexpected empty share name skipped\n"); + next; + } + if ( $shareDuplicate->{$shareName} ) { print(LOG $bpc->timeStamp, "unexpected repeated share name $shareName skipped\n"); next; } + $shareDuplicate->{$shareName} = 1; - if ( $Conf{XferMethod} eq "tar" ) { - # - # Use tar (eg: tar/ssh) as the transport program. - # - $xfer = BackupPC::Xfer::Tar->new($bpc); - } elsif ( $Conf{XferMethod} eq "rsync" || $Conf{XferMethod} eq "rsyncd" ) { - # - # Use rsync as the transport program. - # - if ( !defined($xfer = BackupPC::Xfer::Rsync->new($bpc)) ) { - my $errStr = BackupPC::Xfer::Rsync::errStr; - print(LOG $bpc->timeStamp, "dump failed: $errStr\n"); - print("dump failed: $errStr\n"); - UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); - exit(1); - } - } else { - # - # Default is to use smbclient (smb) as the transport program. - # - $xfer = BackupPC::Xfer::Smb->new($bpc); + UserCommandRun("DumpPreShareCmd", $shareName); + if ( $? && $Conf{UserCmdCheckStatus} ) { + print(LOG $bpc->timeStamp, + "DumpPreShareCmd returned error status $?... exiting\n"); + UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); + $XferLOG->write(\"DumpPreShareCmd returned error status $?... exiting\n"); + $stat{hostError} = "DumpPreShareCmd returned error status $?"; + BackupFailCleanup(); + } + + $xfer = BackupPC::Xfer::create($Conf{XferMethod}, $bpc); + if ( !defined($xfer) ) { + my $errStr = BackupPC::Xfer::errStr(); + print(LOG $bpc->timeStamp, "dump failed: $errStr\n"); + UserCommandRun("DumpPostShareCmd", $shareName) if ( $NeedPostCmd ); + UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); + $XferLOG->write(\"BackupPC::Xfer::create failed: $errStr\n"); + $stat{hostError} = $errStr; + BackupFailCleanup(); } my $useTar = $xfer->useTar; @@ -610,31 +737,30 @@ for my $shareName ( @$ShareNames ) { # Run the transport program # $xfer->args({ - host => $host, - client => $client, - hostIP => $hostIP, - shareName => $shareName, - pipeRH => *RH, - pipeWH => *WH, - XferLOG => $XferLOG, - newFilesFH => $newFilesFH, - outDir => $Dir, - type => $type, - lastFull => $lastFull, - lastBkupNum => $lastBkupNum, - lastFullBkupNum => $lastFullBkupNum, - backups => \@Backups, - compress => $Conf{CompressLevel}, - XferMethod => $Conf{XferMethod}, - logLevel => $Conf{XferLogLevel}, - pidHandler => \&pidHandler, - partialNum => $partialNum, + host => $host, + client => $client, + hostIP => $hostIP, + shareName => $shareName, + pipeRH => *RH, + pipeWH => *WH, + XferLOG => $XferLOG, + newFilesFH => $newFilesFH, + outDir => $Dir, + type => $type, + incrBaseTime => $incrBaseTime, + incrBaseBkupNum => $incrBaseBkupNum, + backups => \@Backups, + compress => $Conf{CompressLevel}, + XferMethod => $Conf{XferMethod}, + logLevel => $Conf{XferLogLevel}, + partialNum => $partialNum, + pidHandler => \&pidHandler, + completionPercent => \&completionPercent, }); if ( !defined($logMsg = $xfer->start()) ) { - print(LOG $bpc->timeStamp, "xfer start failed: ", $xfer->errStr, "\n"); - print("dump failed: ", $xfer->errStr, "\n"); - print("link $clientURI\n") if ( $needLink ); + my $errStr = "xfer start failed: " . $xfer->errStr . "\n"; + print(LOG $bpc->timeStamp, $errStr); # # kill off the tar process, first nicely then forcefully # @@ -648,8 +774,11 @@ for my $shareName ( @$ShareNames ) { sleep(1); kill($bpc->sigName2num("KILL"), @xferPid); } + UserCommandRun("DumpPostShareCmd", $shareName) if ( $NeedPostCmd ); UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); - exit(1); + $XferLOG->write(\"xfer start failed: $errStr\n"); + $stat{hostError} = $errStr; + BackupFailCleanup(); } @xferPid = $xfer->xferPid; @@ -663,6 +792,7 @@ for my $shareName ( @$ShareNames ) { close(WH); } print(LOG $bpc->timeStamp, $logMsg, "\n"); + $XferLOG->write(\"$logMsg\n"); print("started $type dump, share=$shareName\n"); pidHandler(@xferPid); @@ -674,43 +804,41 @@ for my $shareName ( @$ShareNames ) { # we use a select. # my($FDread, $tarOut, $mesg); - vec($FDread, fileno(TAR), 1) = 1 if ( $useTar ); + vec($FDread, fileno(TAR), 1) = 1; $xfer->setSelectMask(\$FDread); SCAN: while ( 1 ) { my $ein = $FDread; last if ( $FDread =~ /^\0*$/ ); select(my $rout = $FDread, undef, $ein, undef); - if ( $useTar ) { - if ( vec($rout, fileno(TAR), 1) ) { - if ( sysread(TAR, $mesg, 8192) <= 0 ) { - vec($FDread, fileno(TAR), 1) = 0; - close(TAR); - } else { - $tarOut .= $mesg; - } - } - while ( $tarOut =~ /(.*?)[\n\r]+(.*)/s ) { - $_ = $1; - $tarOut = $2; - if ( /^ / ) { - $XferLOG->write(\"$_\n"); - } else { - $XferLOG->write(\"tarExtract: $_\n"); - } - if ( /^BackupPC_tarExtact aborting \((.*)\)/ ) { - $stat{hostError} = $1; - } - if ( /^Done: (\d+) errors, (\d+) filesExist, (\d+) sizeExist, (\d+) sizeExistComp, (\d+) filesTotal, (\d+) sizeTotal/ ) { - $tarErrs += $1; - $nFilesExist += $2; - $sizeExist += $3; - $sizeExistComp += $4; - $nFilesTotal += $5; - $sizeTotal += $6; - } - } - } + if ( vec($rout, fileno(TAR), 1) ) { + if ( sysread(TAR, $mesg, 8192) <= 0 ) { + vec($FDread, fileno(TAR), 1) = 0; + close(TAR); + } else { + $tarOut .= $mesg; + } + } + while ( $tarOut =~ /(.*?)[\n\r]+(.*)/s ) { + $_ = $1; + $tarOut = $2; + if ( /^ / ) { + $XferLOG->write(\"$_\n"); + } else { + $XferLOG->write(\"tarExtract: $_\n"); + } + if ( /^BackupPC_tarExtact aborting \((.*)\)/ ) { + $stat{hostError} = $1; + } + if ( /^Done: (\d+) errors, (\d+) filesExist, (\d+) sizeExist, (\d+) sizeExistComp, (\d+) filesTotal, (\d+) sizeTotal/ ) { + $tarErrs += $1; + $nFilesExist += $2; + $sizeExist += $3; + $sizeExistComp += $4; + $nFilesTotal += $5; + $sizeTotal += $6; + } + } last if ( !$xfer->readOutput(\$FDread, $rout) ); while ( my $str = $xfer->logMsgGet ) { print(LOG $bpc->timeStamp, "xfer: $str\n"); @@ -761,10 +889,20 @@ for my $shareName ( @$ShareNames ) { next; } } + + if ( $NeedPostCmd ) { + UserCommandRun("DumpPostShareCmd", $shareName); + if ( $? && $Conf{UserCmdCheckStatus} ) { + print(LOG $bpc->timeStamp, + "DumpPostShareCmd returned error status $?... exiting\n"); + $stat{hostError} = "DumpPostShareCmd returned error status $?"; + } + } + $stat{xferOK} = 0 if ( $stat{hostError} || $stat{hostAbort} ); if ( !$stat{xferOK} ) { # - # kill off the tranfer program, first nicely then forcefully + # kill off the transfer program, first nicely then forcefully # if ( @xferPid ) { kill($bpc->sigName2num("INT"), @xferPid); @@ -784,6 +922,10 @@ for my $shareName ( @$ShareNames ) { # last; } + # + # Wait for any child processes to exit + # + # 1 while ( wait() >= 0 ); } # @@ -797,6 +939,15 @@ if ( $type eq "full" && $stat{hostError} eq "" $stat{xferOK} = 0 if ( $Abort ); +# +# If there is no "new" directory then the backup is bad +# +if ( $stat{xferOK} && !-d "$Dir/new" ) { + $stat{hostError} = "No backup directory $Dir/new" + if ( $stat{hostError} eq "" ); + $stat{xferOK} = 0; +} + # # Do one last check to make sure it is still the machine we expect. # @@ -806,6 +957,12 @@ if ( $stat{xferOK} && (my $errMsg = CorrectHostCheck($hostIP, $host)) ) { } UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); +if ( $? && $Conf{UserCmdCheckStatus} ) { + print(LOG $bpc->timeStamp, + "DumpPostUserCmd returned error status $?... exiting\n"); + $stat{hostError} = "DumpPostUserCmd returned error status $?"; + $stat{xferOK} = 0; +} close($newFilesFH) if ( defined($newFilesFH) ); my $endTime = time(); @@ -847,6 +1004,11 @@ my $newNum = BackupSave(); my $otherCount = $stat{xferErrCnt} - $stat{xferBadFileCnt} - $stat{xferBadShareCnt}; +$stat{fileCnt} ||= 0; +$stat{byteCnt} ||= 0; +$stat{xferErrCnt} ||= 0; +$stat{xferBadFileCnt} ||= 0; +$stat{xferBadShareCnt} ||= 0; print(LOG $bpc->timeStamp, "$type backup $newNum complete, $stat{fileCnt} files," . " $stat{byteCnt} bytes," @@ -855,6 +1017,8 @@ print(LOG $bpc->timeStamp, BackupExpire($client); +UserCommandRun("DumpPostFinishCmd"); # XXX ASA search integration + print("$type backup complete\n"); ########################################################################### @@ -895,9 +1059,9 @@ sub catch_signal "Aborting backup up after signal $sigName\n"); # - # Tell xfer to abort + # Tell xfer to abort, but only if we actually started one # - $xfer->abort($reason); + $xfer->abort($reason) if ( defined($xfer) ); # # Send ALRMs to BackupPC_tarExtract if we are using it @@ -990,7 +1154,9 @@ sub BackupFailCleanup # # We keep this backup if it is a full and we actually backed - # up some files. + # up some files. If the prior backup was a partial too, we + # only keep this backup if it has more files than the previous + # partial. # if ( $type eq "full" ) { if ( $nFilesTotal == 0 && $xfer->getStats->{fileCnt} == 0 ) { @@ -999,15 +1165,36 @@ sub BackupFailCleanup # directory just in case. # find(\&CheckForNewFiles, "$Dir/new"); - $keepPartial = 1 if ( $nFilesTotal ); - } else { - # - # Xfer reported some files - # + } + my $str; + if ( $nFilesTotal > $partialFileCnt + || $xfer->getStats->{fileCnt} > $partialFileCnt ) { + # + # If the last backup wasn't a partial then + # $partialFileCnt is undefined, so the above + # test is simply $nFilesTotal > 0 + # $keepPartial = 1; - } + if ( $partialFileCnt ) { + $str = "Saving this as a partial backup\n"; + } else { + $str = sprintf("Saving this as a partial backup, replacing the" + . " prior one (got %d and %d files versus %d)\n", + $nFilesTotal, $xfer->getStats->{fileCnt}, $partialFileCnt); + } + } else { + $str = sprintf("Not saving this as a partial backup since it has fewer" + . " files than the prior one (got %d and %d files versus %d)\n", + $nFilesTotal, $xfer->getStats->{fileCnt}, $partialFileCnt); + } + $XferLOG->write(\$str); } + # + # Don't keep partials if they are disabled + # + $keepPartial = 0 if ( $Conf{PartialAgeMax} < 0 ); + if ( !$keepPartial ) { # # No point in saving this dump; get rid of eveything. @@ -1046,7 +1233,8 @@ sub BackupExpire my($client) = @_; my($Dir) = "$TopDir/pc/$client"; my(@Backups) = $bpc->BackupInfoRead($client); - my($cntFull, $cntIncr, $firstFull, $firstIncr, $oldestIncr, $oldestFull); + my($cntFull, $cntIncr, $firstFull, $firstIncr, $oldestIncr, + $oldestFull, $changes); if ( $Conf{FullKeepCnt} <= 0 ) { print(LOG $bpc->timeStamp, @@ -1063,7 +1251,7 @@ sub BackupExpire if ( $Backups[$i]{type} eq "full" ) { $firstFull = $i if ( $cntFull == 0 ); $cntFull++; - } else { + } elsif ( $Backups[$i]{type} eq "incr" ) { $firstIncr = $i if ( $cntIncr == 0 ); $cntIncr++; } @@ -1072,22 +1260,39 @@ sub BackupExpire if ( $cntIncr > 0 ); $oldestFull = (time - $Backups[$firstFull]{startTime}) / (24 * 3600) if ( $cntFull > 0 ); - if ( $cntIncr > $Conf{IncrKeepCnt} - || ($cntIncr > $Conf{IncrKeepCntMin} - && $oldestIncr > $Conf{IncrAgeMax}) - && (@Backups <= $firstIncr + 1 - || $Backups[$firstIncr]{noFill} - || !$Backups[$firstIncr + 1]{noFill}) ) { + + # + # With multi-level incrementals, several of the following + # incrementals might depend upon this one, so we have to + # delete all of the them. Figure out if that is possible + # by counting the number of consecutive incrementals that + # are unfilled and have a level higher than this one. + # + my $cntIncrDel = 1; + my $earliestIncr = $oldestIncr; + + for ( my $i = $firstIncr + 1 ; $i < @Backups ; $i++ ) { + last if ( $Backups[$i]{level} <= $Backups[$firstIncr]{level} + || !$Backups[$i]{noFill} ); + $cntIncrDel++; + $earliestIncr = (time - $Backups[$i]{startTime}) / (24 * 3600); + } + + if ( $cntIncr >= $Conf{IncrKeepCnt} + $cntIncrDel + || ($cntIncr >= $Conf{IncrKeepCntMin} + $cntIncrDel + && $earliestIncr > $Conf{IncrAgeMax}) ) { # - # Only delete an incr backup if the Conf settings are satisfied. - # We also must make sure that either this backup is the most - # recent one, or it is not filled, or the next backup is filled. - # (We can't deleted a filled incr if the next backup is not - # filled.) + # Only delete an incr backup if the Conf settings are satisfied + # for all $cntIncrDel incrementals. Since BackupRemove() does + # a splice() we need to do the deletes in the reverse order. # - print(LOG $bpc->timeStamp, - "removing incr backup $Backups[$firstIncr]{num}\n"); - BackupRemove($client, \@Backups, $firstIncr); + for ( my $i = $firstIncr + $cntIncrDel - 1 ; + $i >= $firstIncr ; $i-- ) { + print(LOG $bpc->timeStamp, + "removing incr backup $Backups[$i]{num}\n"); + BackupRemove($client, \@Backups, $i); + $changes++; + } next; } @@ -1095,8 +1300,26 @@ sub BackupExpire # Delete any old full backups, according to $Conf{FullKeepCntMin} # and $Conf{FullAgeMax}. # + # First make sure that $Conf{FullAgeMax} is at least bigger + # than $Conf{FullPeriod} * $Conf{FullKeepCnt}, including + # the exponential array case. + # + my $fullKeepCnt = $Conf{FullKeepCnt}; + $fullKeepCnt = [$fullKeepCnt] if ( ref($fullKeepCnt) ne "ARRAY" ); + my $fullAgeMax; + my $fullPeriod = int(0.5 + $Conf{FullPeriod}); + $fullPeriod = 7 if ( $fullPeriod <= 0 ); + for ( my $i = 0 ; $i < @$fullKeepCnt ; $i++ ) { + $fullAgeMax += $fullKeepCnt->[$i] * $fullPeriod; + $fullPeriod *= 2; + } + $fullAgeMax += $fullPeriod; # add some buffer + if ( $cntFull > $Conf{FullKeepCntMin} && $oldestFull > $Conf{FullAgeMax} + && $oldestFull > $fullAgeMax + && $Conf{FullKeepCntMin} > 0 + && $Conf{FullAgeMax} > 0 && (@Backups <= $firstFull + 1 || !$Backups[$firstFull + 1]{noFill}) ) { # @@ -1109,6 +1332,7 @@ sub BackupExpire print(LOG $bpc->timeStamp, "removing old full backup $Backups[$firstFull]{num}\n"); BackupRemove($client, \@Backups, $firstFull); + $changes++; next; } @@ -1117,8 +1341,9 @@ sub BackupExpire # where $Conf{FullKeepCnt} is an array. # last if ( !BackupFullExpire($client, \@Backups) ); + $changes++; } - $bpc->BackupInfoWrite($client, @Backups); + $bpc->BackupInfoWrite($client, @Backups) if ( $changes ); } # @@ -1172,8 +1397,8 @@ sub BackupFullExpire # # Delete the full backup # - #printf("Deleting backup $i ($prevFull)\n"); - push(@delete, $i); + #print("Deleting backup $i ($prevFull)\n"); + unshift(@delete, $i); } else { $fullCnt++; while ( $fullKeepIdx < @$fullKeepCnt @@ -1212,12 +1437,14 @@ sub BackupSave { my @Backups = $bpc->BackupInfoRead($client); my $num = -1; + my $newFilesFH; # # Since we got a good backup we should remove any partial dumps # (the new backup might also be a partial, but that's ok). # BackupPartialRemove($client, \@Backups); + $needLink = 1 if ( -f "$Dir/NewFileList" ); # # Number the new backup @@ -1230,8 +1457,8 @@ sub BackupSave if ( !rename("$Dir/new", "$Dir/$num") ) { print(LOG $bpc->timeStamp, "Rename $Dir/new -> $Dir/$num failed\n"); $stat{xferOK} = 0; + return; } - $needLink = 1 if ( -f "$Dir/NewFileList" ); # # Add the new backup information to the backup file @@ -1252,9 +1479,21 @@ sub BackupSave $Backups[$i]{tarErrs} = $tarErrs; $Backups[$i]{compress} = $Conf{CompressLevel}; $Backups[$i]{noFill} = $type eq "incr" ? 1 : 0; - $Backups[$i]{level} = $type eq "incr" ? 1 : 0; - $Backups[$i]{mangle} = 1; # name mangling always on for v1.04+ + $Backups[$i]{level} = $incrLevel; + $Backups[$i]{mangle} = 1; # name mangling always on for v1.04+ + $Backups[$i]{xferMethod} = $Conf{XferMethod}; + $Backups[$i]{charset} = $Conf{ClientCharset}; + $Backups[$i]{version} = $bpc->Version(); + # + # Save the main backups file + # $bpc->BackupInfoWrite($client, @Backups); + # + # Save just this backup's info in case the main backups file + # gets corrupted + # + BackupPC::Storage->backupInfoWrite($Dir, $Backups[$i]{num}, + $Backups[$i]); unlink("$Dir/timeStamp.level0") if ( -f "$Dir/timeStamp.level0" ); foreach my $ext ( qw(bad bad.z) ) { @@ -1280,11 +1519,34 @@ sub BackupSave $file = "$f->{share}/$f->{file}"; } next if ( !-f "$Dir/$Backups[$j]{num}/$file" ); - if ( !link("$Dir/$Backups[$j]{num}/$file", - "$Dir/$num/$shareM/$fileM") ) { - my $str = \"Unable to link $num/$f->{share}/$f->{file} to" - . " $Backups[$j]{num}/$f->{share}/$f->{file}\n"; - $XferLOG->write(\$str); + + my($exists, $digest, $origSize, $outSize, $errs) + = BackupPC::PoolWrite::LinkOrCopy( + $bpc, + "$Dir/$Backups[$j]{num}/$file", + $Backups[$j]{compress}, + "$Dir/$num/$shareM/$fileM", + $Conf{CompressLevel}); + if ( !$exists ) { + # + # the hard link failed, most likely because the target + # file has too many links. We have copied the file + # instead, so add this to the new file list. + # + if ( !defined($newFilesFH) ) { + my $str = "Appending to NewFileList for $shareM/$fileM\n"; + $XferLOG->write(\$str); + open($newFilesFH, ">>", "$TopDir/pc/$client/NewFileList") + || die("can't open $TopDir/pc/$client/NewFileList"); + binmode($newFilesFH); + } + if ( -f "$Dir/$num/$shareM/$fileM" ) { + print($newFilesFH "$digest $origSize $shareM/$fileM\n"); + } else { + my $str = "Unable to link/copy $num/$f->{share}/$f->{file}" + . " to $Backups[$j]{num}/$f->{share}/$f->{file}\n"; + $XferLOG->write(\$str); + } } else { my $str = "Bad file $num/$f->{share}/$f->{file} replaced" . " by link to" @@ -1299,6 +1561,7 @@ sub BackupSave $XferLOG->write(\$str); } } + close($newFilesFH) if ( defined($newFilesFH) ); $XferLOG->close(); rename("$Dir/XferLOG$fileExt", "$Dir/XferLOG.$num$fileExt"); rename("$Dir/NewFileList", "$Dir/NewFileList.$num"); @@ -1314,6 +1577,11 @@ sub BackupRemove my($client, $Backups, $idx) = @_; my($Dir) = "$TopDir/pc/$client"; + if ( $Backups->[$idx]{num} eq "" ) { + print("BackupRemove: ignoring empty backup number for idx $idx\n"); + return; + } + $bpc->RmTreeDefer("$TopDir/trash", "$Dir/$Backups->[$idx]{num}"); unlink("$Dir/SmbLOG.$Backups->[$idx]{num}") @@ -1334,7 +1602,7 @@ sub CorrectHostCheck || $Conf{NmbLookupCmd} eq "" ); my($netBiosHost, $netBiosUser) = $bpc->NetBiosInfoGet($hostIP); return "host $host has mismatching netbios name $netBiosHost" - if ( $netBiosHost ne $host ); + if ( lc($netBiosHost) ne lc(substr($host, 0, 15)) ); return; } @@ -1355,14 +1623,31 @@ sub pidHandler print("xferPids $str\n"); } +# +# The Xfer method might tell us from time to time about progress +# in the backup or restore +# +sub completionPercent +{ + my($percent) = @_; + + $percent = 100 if ( $percent > 100 ); + $percent = 0 if ( $percent < 0 ); + if ( !defined($completionPercent) + || int($completionPercent + 0.5) != int($percent) ) { + printf("completionPercent %.0f\n", $percent); + } + $completionPercent = $percent; +} + # # Run an optional pre- or post-dump command # sub UserCommandRun { - my($type) = @_; + my($cmdType, $sharename) = @_; - return if ( !defined($Conf{$type}) ); + return if ( !defined($Conf{$cmdType}) ); my $vars = { xfer => $xfer, client => $client, @@ -1380,9 +1665,15 @@ sub UserCommandRun xferOK => $stat{xferOK} || 0, hostError => $stat{hostError}, type => $type, + cmdType => $cmdType, }; - my $cmd = $bpc->cmdVarSubstitute($Conf{$type}, $vars); - $XferLOG->write(\"Executing $type: @$cmd\n"); + + if ($cmdType eq 'DumpPreShareCmd' || $cmdType eq 'DumpPostShareCmd') { + $vars->{share} = $sharename; + } + + my $cmd = $bpc->cmdVarSubstitute($Conf{$cmdType}, $vars); + $XferLOG->write(\"Executing $cmdType: @$cmd\n"); # # Run the user's command, dumping the stdout/stderr into the # Xfer log file. Also supply the optional $vars and %Conf in @@ -1392,6 +1683,7 @@ sub UserCommandRun $bpc->cmdSystemOrEval($cmd, sub { $XferLOG->write(\$_[0]); + print(LOG $bpc->timeStamp, "Output from $cmdType: ", $_[0]); }, $vars, \%Conf); }