X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=bin%2FBackupPC_dump;h=72f992c14432223f9462ac02343bea7fe10e118d;hp=79393bf422c043b9e474602bb4d405e51b69eec6;hb=5b3e6091d542c2e7445d5dd511cdf6e20aec8b8d;hpb=329e870f56fb6572fa697998d33676588034c149 diff --git a/bin/BackupPC_dump b/bin/BackupPC_dump index 79393bf..72f992c 100755 --- a/bin/BackupPC_dump +++ b/bin/BackupPC_dump @@ -1,4 +1,4 @@ -#!/bin/perl -T +#!/bin/perl #============================================================= -*-perl-*- # # BackupPC_dump: Dump a single client. @@ -31,9 +31,9 @@ # full or incremental backup needs to be run. If no backup is # scheduled, or a ping to $client fails, then BackupPC_dump quits. # -# The backup is done using the selected XferMethod (smb, tar, rsync etc), -# extracting the dump into $TopDir/pc/$client/new. The xfer output is -# put into $TopDir/pc/$client/XferLOG. +# The backup is done using the selected XferMethod (smb, tar, rsync, +# backuppcd etc), extracting the dump into $TopDir/pc/$client/new. +# The xfer output is put into $TopDir/pc/$client/XferLOG. # # If the dump succeeds (based on parsing the output of the XferMethod): # - $TopDir/pc/$client/new is renamed to $TopDir/pc/$client/nnn, where @@ -70,7 +70,7 @@ # #======================================================================== # -# Version 2.1.0_CVS, released 3 Jul 2003. +# Version 3.0.0alpha, released 23 Jan 2006. # # See http://backuppc.sourceforge.net. # @@ -81,11 +81,14 @@ no utf8; use lib "/usr/local/BackupPC/lib"; use BackupPC::Lib; use BackupPC::FileZIO; +use BackupPC::Storage; use BackupPC::Xfer::Smb; use BackupPC::Xfer::Tar; use BackupPC::Xfer::Rsync; +use BackupPC::Xfer::BackupPCd; use Socket; use File::Path; +use File::Find; use Getopt::Std; ########################################################################### @@ -98,6 +101,8 @@ my $BinDir = $bpc->BinDir(); my %Conf = $bpc->Conf(); my $NeedPostCmd; my $Hosts; +my $SigName; +my $Abort; $bpc->ChildInit(); @@ -126,20 +131,19 @@ if ( $opts{d} ) { # $hostIP = $client; if ( $bpc->CheckHostAlive($hostIP) < 0 ) { - print("Exiting because CheckHostAlive($hostIP) failed\n") + print(STDERR "Exiting because CheckHostAlive($hostIP) failed\n") if ( $opts{v} ); exit(1); } if ( $Conf{NmbLookupCmd} eq "" ) { - print("Exiting because \$Conf{NmbLookupCmd} is empty\n") + print(STDERR "Exiting because \$Conf{NmbLookupCmd} is empty\n") if ( $opts{v} ); exit(1); } ($client, $user) = $bpc->NetBiosInfoGet($hostIP); if ( $client !~ /^([\w\.\s-]+)$/ ) { - print("Exiting because NetBiosInfoGet($hostIP) returned '$client'," - . " an invalid host name\n") - if ( $opts{v} ); + print(STDERR "Exiting because NetBiosInfoGet($hostIP) returned" + . " '$client', an invalid host name\n") if ( $opts{v} ); exit(1) } $Hosts = $bpc->HostInfoRead($client); @@ -148,8 +152,8 @@ if ( $opts{d} ) { $Hosts = $bpc->HostInfoRead($client); } if ( !defined($Hosts->{$client}) ) { - print("Exiting because host $client does not exist in the hosts file\n") - if ( $opts{v} ); + print(STDERR "Exiting because host $client does not exist in the" + . " hosts file\n") if ( $opts{v} ); exit(1) } @@ -200,6 +204,15 @@ if ( $opts{e} ) { exit(0); } +# +# For archive hosts we don't bother any further +# +if ($Conf{XferMethod} eq "archive" ) { + print(STDERR "Exiting because the XferMethod is set to archive\n") + if ( $opts{v} ); + exit(0); +} + if ( !$opts{d} ) { # # In the non-DHCP case, make sure the host can be looked up @@ -215,7 +228,7 @@ if ( !$opts{d} ) { # Ok, NS doesn't know about it. Maybe it is a NetBios name # instead. # - print("Name server doesn't know about $host; trying NetBios\n") + print(STDERR "Name server doesn't know about $host; trying NetBios\n") if ( $opts{v} ); if ( !defined($hostIP = $bpc->NetBiosHostIPFind($host)) ) { print(LOG $bpc->timeStamp, "Can't find host $host via netbios\n"); @@ -253,7 +266,7 @@ $bpc->ServerDisconnect(); if ( $opts{d} ) { if ( $StatusHost{activeJob} ) { # oops, something is already running for this host - print("Exiting because backup is already running for $client\n") + print(STDERR "Exiting because backup is already running for $client\n") if ( $opts{v} ); exit(0); } @@ -263,9 +276,22 @@ if ( $opts{d} ) { my($needLink, @Backups, $type, $lastBkupNum, $lastFullBkupNum); my $lastFull = 0; my $lastIncr = 0; +my $partialIdx = -1; +my $partialNum; +my $lastPartial = 0; if ( $Conf{FullPeriod} == -1 && !$opts{f} && !$opts{i} || $Conf{FullPeriod} == -2 ) { + print(STDERR "Exiting because backups are disabled with" + . " \$Conf{FullPeriod} = $Conf{FullPeriod}\n") if ( $opts{v} ); + # + # Tell BackupPC to ignore old failed backups on hosts that + # have backups disabled. + # + print("backups disabled\n") + if ( defined($StatusHost{errorTime}) + && $StatusHost{reason} ne "Reason_backup_done" + && time - $StatusHost{errorTime} > 4 * 24 * 3600 ); NothingToDo($needLink); } @@ -273,17 +299,58 @@ if ( !$opts{i} && !$opts{f} && $Conf{BlackoutGoodCnt} >= 0 && $StatusHost{aliveCnt} >= $Conf{BlackoutGoodCnt} ) { my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); my($currHours) = $hour + $min / 60 + $sec / 3600; - if ( $Conf{BlackoutHourBegin} <= $currHours - && $currHours <= $Conf{BlackoutHourEnd} - && grep($_ == $wday, @{$Conf{BlackoutWeekDays}}) ) { -# print(LOG $bpc->timeStamp, "skipping because of blackout" -# . " (alive $StatusHost{aliveCnt} times)\n"); - NothingToDo($needLink); + my $blackout; + + # + # Handle backward compatibility with original separate scalar + # parameters. + # + if ( defined($Conf{BlackoutHourBegin}) ) { + push(@{$Conf{BlackoutPeriods}}, + { + hourBegin => $Conf{BlackoutHourBegin}, + hourEnd => $Conf{BlackoutHourEnd}, + weekDays => $Conf{BlackoutWeekDays}, + } + ); + } + foreach my $p ( @{$Conf{BlackoutPeriods}} ) { + # + # Allow blackout to span midnight (specified by BlackoutHourBegin + # being greater than BlackoutHourEnd) + # + next if ( ref($p->{weekDays}) ne "ARRAY" + || !defined($p->{hourBegin}) + || !defined($p->{hourEnd}) + ); + if ( $p->{hourBegin} > $p->{hourEnd} ) { + $blackout = $p->{hourBegin} <= $currHours + || $currHours <= $p->{hourEnd}; + if ( $currHours <= $p->{hourEnd} ) { + # + # This is after midnight, so decrement the weekday for the + # weekday check (eg: Monday 11pm-1am means Monday 2300 to + # Tuesday 0100, not Monday 2300-2400 plus Monday 0000-0100). + # + $wday--; + $wday += 7 if ( $wday < 0 ); + } + } else { + $blackout = $p->{hourBegin} <= $currHours + && $currHours <= $p->{hourEnd}; + } + if ( $blackout && grep($_ == $wday, @{$p->{weekDays}}) ) { +# print(LOG $bpc->timeStamp, "skipping because of blackout" +# . " (alive $StatusHost{aliveCnt} times)\n"); + print(STDERR "Skipping $client because of blackout\n") + if ( $opts{v} ); + NothingToDo($needLink); + } } } if ( !$opts{i} && !$opts{f} && $StatusHost{backoffTime} > time ) { - printf(LOG "%sskipping because of user requested delay (%.1f hours left)", + printf(LOG "%sskipping because of user requested delay (%.1f hours left)\n", $bpc->timeStamp, ($StatusHost{backoffTime} - time) / 3600); NothingToDo($needLink); } @@ -307,9 +374,13 @@ for ( my $i = 0 ; $i < @Backups ; $i++ ) { $lastFull = $Backups[$i]{startTime}; $lastFullBkupNum = $Backups[$i]{num}; } - } else { + } elsif ( $Backups[$i]{type} eq "incr" ) { $lastIncr = $Backups[$i]{startTime} if ( $lastIncr < $Backups[$i]{startTime} ); + } elsif ( $Backups[$i]{type} eq "partial" ) { + $partialIdx = $i; + $lastPartial = $Backups[$i]{startTime}; + $partialNum = $Backups[$i]{num}; } } @@ -365,7 +436,11 @@ $bpc->RmTreeDefer("$TopDir/trash", "$Dir/new") if ( -d "$Dir/new" ); # # Setup file extension for compression and open XferLOG output file # -$Conf{CompressLevel} = 0 if ( !BackupPC::FileZIO->compOk ); +if ( $Conf{CompressLevel} && !BackupPC::FileZIO->compOk ) { + print(LOG $bpc->timeStamp, "dump failed: can't find Compress::Zlib\n"); + print("dump failed: can't find Compress::Zlib\n"); + exit(1); +} my $fileExt = $Conf{CompressLevel} > 0 ? ".z" : ""; my $XferLOG = BackupPC::FileZIO->open("$Dir/XferLOG$fileExt", 1, $Conf{CompressLevel}); @@ -375,23 +450,56 @@ if ( !defined($XferLOG) ) { print("dump failed: unable to open/create $Dir/XferLOG$fileExt\n"); exit(1); } -$XferLOG->writeTeeStdout(1) if ( $opts{v} ); -unlink("$Dir/NewFileList"); -my $startTime = time(); +# +# Ignore the partial dump in the case of an incremental +# or when the partial is too old. A partial is a partial full. +# +if ( $type ne "full" || time - $lastPartial > $Conf{PartialAgeMax} * 24*3600 ) { + $partialNum = undef; + $partialIdx = -1; +} + +# +# If this is a partial, copy the old XferLOG file +# +if ( $partialNum ) { + my($compress, $fileName); + if ( -f "$Dir/XferLOG.$partialNum.z" ) { + $fileName = "$Dir/XferLOG.$partialNum.z"; + $compress = 1; + } elsif ( -f "$Dir/XferLOG.$partialNum" ) { + $fileName = "$Dir/XferLOG.$partialNum"; + $compress = 0; + } + if ( my $oldLOG = BackupPC::FileZIO->open($fileName, 0, $compress) ) { + my $data; + while ( $oldLOG->read(\$data, 65536) > 0 ) { + $XferLOG->write(\$data); + } + $oldLOG->close; + } +} + +$XferLOG->writeTeeStderr(1) if ( $opts{v} ); +unlink("$Dir/NewFileList") if ( -f "$Dir/NewFileList" ); + +my $startTime = time(); my $tarErrs = 0; my $nFilesExist = 0; my $sizeExist = 0; my $sizeExistComp = 0; my $nFilesTotal = 0; my $sizeTotal = 0; -my($logMsg, %stat, $xfer, $ShareNames); +my($logMsg, %stat, $xfer, $ShareNames, $noFilesErr); my $newFilesFH; if ( $Conf{XferMethod} eq "tar" ) { $ShareNames = $Conf{TarShareName}; } elsif ( $Conf{XferMethod} eq "rsync" || $Conf{XferMethod} eq "rsyncd" ) { $ShareNames = $Conf{RsyncShareName}; +} elsif ( $Conf{XferMethod} eq "backuppcd" ) { + $ShareNames = $Conf{BackupPCdShareName}; } else { $ShareNames = $Conf{SmbShareName}; } @@ -418,6 +526,8 @@ for my $shareName ( @$ShareNames ) { next; } + UserCommandRun("DumpPreShareCmd", $shareName); + if ( $Conf{XferMethod} eq "tar" ) { # # Use tar (eg: tar/ssh) as the transport program. @@ -431,6 +541,19 @@ for my $shareName ( @$ShareNames ) { my $errStr = BackupPC::Xfer::Rsync::errStr; print(LOG $bpc->timeStamp, "dump failed: $errStr\n"); print("dump failed: $errStr\n"); + UserCommandRun("DumpPostShareCmd", $shareName) if ( $NeedPostCmd ); + UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); + exit(1); + } + } elsif ( $Conf{XferMethod} eq "backuppcd" ) { + # + # Use backuppcd as the transport program. + # + if ( !defined($xfer = BackupPC::Xfer::BackupPCd->new($bpc)) ) { + my $errStr = BackupPC::Xfer::BackupPCd::errStr; + print(LOG $bpc->timeStamp, "dump failed: $errStr\n"); + print("dump failed: $errStr\n"); + UserCommandRun("DumpPostShareCmd", $shareName) if ( $NeedPostCmd ); UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); exit(1); } @@ -490,6 +613,7 @@ for my $shareName ( @$ShareNames ) { open(STDERR, ">&STDOUT"); close(STDIN); open(STDIN, "<&RH"); + alarm(0); exec("$BinDir/BackupPC_tarExtract", $client, $shareName, $Conf{CompressLevel}); print(LOG $bpc->timeStamp, @@ -527,7 +651,9 @@ for my $shareName ( @$ShareNames ) { backups => \@Backups, compress => $Conf{CompressLevel}, XferMethod => $Conf{XferMethod}, + logLevel => $Conf{XferLogLevel}, pidHandler => \&pidHandler, + partialNum => $partialNum, }); if ( !defined($logMsg = $xfer->start()) ) { @@ -538,15 +664,16 @@ for my $shareName ( @$ShareNames ) { # kill off the tar process, first nicely then forcefully # if ( $tarPid > 0 ) { - kill(2, $tarPid); + kill($bpc->sigName2num("INT"), $tarPid); sleep(1); - kill(9, $tarPid); + kill($bpc->sigName2num("KILL"), $tarPid); } if ( @xferPid ) { - kill(2, @xferPid); + kill($bpc->sigName2num("INT"), @xferPid); sleep(1); - kill(9, @xferPid); + kill($bpc->sigName2num("KILL"), @xferPid); } + UserCommandRun("DumpPostShareCmd", $shareName) if ( $NeedPostCmd ); UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); exit(1); } @@ -592,7 +719,14 @@ for my $shareName ( @$ShareNames ) { while ( $tarOut =~ /(.*?)[\n\r]+(.*)/s ) { $_ = $1; $tarOut = $2; - $XferLOG->write(\"tarExtract: $_\n"); + if ( /^ / ) { + $XferLOG->write(\"$_\n"); + } else { + $XferLOG->write(\"tarExtract: $_\n"); + } + if ( /^BackupPC_tarExtact aborting \((.*)\)/ ) { + $stat{hostError} = $1; + } if ( /^Done: (\d+) errors, (\d+) filesExist, (\d+) sizeExist, (\d+) sizeExistComp, (\d+) filesTotal, (\d+) sizeTotal/ ) { $tarErrs += $1; $nFilesExist += $2; @@ -637,6 +771,9 @@ for my $shareName ( @$ShareNames ) { # Merge the xfer status (need to accumulate counts) # my $newStat = $xfer->getStats; + if ( $newStat->{fileCnt} == 0 ) { + $noFilesErr ||= "No files dumped for share $shareName"; + } foreach my $k ( (keys(%stat), keys(%$newStat)) ) { next if ( !defined($newStat->{$k}) ); if ( $k =~ /Cnt$/ ) { @@ -650,23 +787,26 @@ for my $shareName ( @$ShareNames ) { next; } } + + UserCommandRun("DumpPostShareCmd", $shareName) if ( $NeedPostCmd ); + $stat{xferOK} = 0 if ( $stat{hostError} || $stat{hostAbort} ); if ( !$stat{xferOK} ) { # # kill off the tranfer program, first nicely then forcefully # if ( @xferPid ) { - kill(2, @xferPid); + kill($bpc->sigName2num("INT"), @xferPid); sleep(1); - kill(9, @xferPid); + kill($bpc->sigName2num("KILL"), @xferPid); } # # kill off the tar process, first nicely then forcefully # if ( $tarPid > 0 ) { - kill(2, $tarPid); + kill($bpc->sigName2num("INT"), $tarPid); sleep(1); - kill(9, $tarPid); + kill($bpc->sigName2num("KILL"), $tarPid); } # # don't do any more shares on this host @@ -674,7 +814,17 @@ for my $shareName ( @$ShareNames ) { last; } } -my $lastNum = -1; + +# +# If this is a full, and any share had zero files then consider the dump bad +# +if ( $type eq "full" && $stat{hostError} eq "" + && length($noFilesErr) && $Conf{BackupZeroFilesIsFatal} ) { + $stat{hostError} = $noFilesErr; + $stat{xferOK} = 0; +} + +$stat{xferOK} = 0 if ( $Abort ); # # Do one last check to make sure it is still the machine we expect. @@ -685,116 +835,49 @@ if ( $stat{xferOK} && (my $errMsg = CorrectHostCheck($hostIP, $host)) ) { } UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); -$XferLOG->close(); close($newFilesFH) if ( defined($newFilesFH) ); -if ( $stat{xferOK} ) { - @Backups = $bpc->BackupInfoRead($client); - for ( my $i = 0 ; $i < @Backups ; $i++ ) { - $lastNum = $Backups[$i]{num} if ( $lastNum < $Backups[$i]{num} ); - } - $lastNum++; - $bpc->RmTreeDefer("$TopDir/trash", "$Dir/$lastNum") - if ( -d "$Dir/$lastNum" ); - if ( !rename("$Dir/new", "$Dir/$lastNum") ) { - print(LOG $bpc->timeStamp, - "Rename $Dir/new -> $Dir/$lastNum failed\n"); - $stat{xferOK} = 0; - } - rename("$Dir/XferLOG$fileExt", "$Dir/XferLOG.$lastNum$fileExt"); - rename("$Dir/NewFileList", "$Dir/NewFileList.$lastNum"); -} my $endTime = time(); # # If the dump failed, clean up # if ( !$stat{xferOK} ) { - # - # wait a short while and see if the system is still alive - # $stat{hostError} = $stat{lastOutputLine} if ( $stat{hostError} eq "" ); if ( $stat{hostError} ) { print(LOG $bpc->timeStamp, "Got fatal error during xfer ($stat{hostError})\n"); + $XferLOG->write(\"Got fatal error during xfer ($stat{hostError})\n"); } - sleep(10); - if ( $bpc->CheckHostAlive($hostIP) < 0 ) { - $stat{hostAbort} = 1; - } - if ( $stat{hostAbort} ) { - $stat{hostError} = "lost network connection during backup"; - } - print(LOG $bpc->timeStamp, "Dump aborted ($stat{hostError})\n"); - unlink("$Dir/timeStamp.level0"); - unlink("$Dir/SmbLOG.bad"); - unlink("$Dir/SmbLOG.bad$fileExt"); - unlink("$Dir/XferLOG.bad"); - unlink("$Dir/XferLOG.bad$fileExt"); - unlink("$Dir/NewFileList"); - rename("$Dir/XferLOG$fileExt", "$Dir/XferLOG.bad$fileExt"); - $bpc->RmTreeDefer("$TopDir/trash", "$Dir/new") if ( -d "$Dir/new" ); - print("dump failed: $stat{hostError}\n"); - print("link $clientURI\n") if ( $needLink ); - exit(1); -} - -# -# Add the new backup information to the backup file -# -@Backups = $bpc->BackupInfoRead($client); -my $i = @Backups; -$Backups[$i]{num} = $lastNum; -$Backups[$i]{type} = $type; -$Backups[$i]{startTime} = $startTime; -$Backups[$i]{endTime} = $endTime; -$Backups[$i]{size} = $sizeTotal; -$Backups[$i]{nFiles} = $nFilesTotal; -$Backups[$i]{xferErrs} = $stat{xferErrCnt} || 0; -$Backups[$i]{xferBadFile} = $stat{xferBadFileCnt} || 0; -$Backups[$i]{xferBadShare} = $stat{xferBadShareCnt} || 0; -$Backups[$i]{nFilesExist} = $nFilesExist; -$Backups[$i]{sizeExist} = $sizeExist; -$Backups[$i]{sizeExistComp} = $sizeExistComp; -$Backups[$i]{tarErrs} = $tarErrs; -$Backups[$i]{compress} = $Conf{CompressLevel}; -$Backups[$i]{noFill} = $type eq "full" ? 0 : 1; -$Backups[$i]{mangle} = 1; # name mangling always on for v1.04+ -$bpc->BackupInfoWrite($client, @Backups); - -unlink("$Dir/timeStamp.level0"); - -# -# Now remove the bad files, replacing them if possible with links to -# earlier backups. -# -foreach my $file ( $xfer->getBadFiles ) { - my $j; - unlink("$Dir/$lastNum/$file"); - for ( $j = $i - 1 ; $j >= 0 ; $j-- ) { - next if ( !-f "$Dir/$Backups[$j]{num}/$file" ); - if ( !link("$Dir/$Backups[$j]{num}/$file", "$Dir/$lastNum/$file") ) { - print(LOG $bpc->timeStamp, - "Unable to link $lastNum/$file to" - . " $Backups[$j]{num}/$file\n"); - } else { - print(LOG $bpc->timeStamp, - "Bad file $lastNum/$file replaced by link to" - . " $Backups[$j]{num}/$file\n"); - } - last; - } - if ( $j < 0 ) { - print(LOG $bpc->timeStamp, - "Removed bad file $lastNum/$file (no older" - . " copy to link to)\n"); + if ( !$Abort ) { + # + # wait a short while and see if the system is still alive + # + sleep(5); + if ( $bpc->CheckHostAlive($hostIP) < 0 ) { + $stat{hostAbort} = 1; + } + if ( $stat{hostAbort} ) { + $stat{hostError} = "lost network connection during backup"; + } + print(LOG $bpc->timeStamp, "Backup aborted ($stat{hostError})\n"); + $XferLOG->write(\"Backup aborted ($stat{hostError})\n"); + } else { + $XferLOG->write(\"Backup aborted by user signal\n"); } + + # + # Close the log file and call BackupFailCleanup, which exits. + # + BackupFailCleanup(); } +my $newNum = BackupSave(); + my $otherCount = $stat{xferErrCnt} - $stat{xferBadFileCnt} - $stat{xferBadShareCnt}; print(LOG $bpc->timeStamp, - "$type backup $lastNum complete, $stat{fileCnt} files," + "$type backup $newNum complete, $stat{fileCnt} files," . " $stat{byteCnt} bytes," . " $stat{xferErrCnt} xferErrs ($stat{xferBadFileCnt} bad files," . " $stat{xferBadShareCnt} bad shares, $otherCount other)\n"); @@ -818,47 +901,174 @@ sub NothingToDo sub catch_signal { - my $signame = shift; - my $fileExt = $Conf{CompressLevel} > 0 ? ".z" : ""; + my $sigName = shift; # - # Children quit quietly on ALRM + # The first time we receive a signal we try to gracefully + # abort the backup. This allows us to keep a partial dump + # with the in-progress file deleted and attribute caches + # flushed to disk etc. # - exit(1) if ( $Pid != $$ && $signame eq "ALRM" ); + if ( !length($SigName) ) { + my $reason; + if ( $sigName eq "INT" ) { + $reason = "aborted by user (signal=$sigName)"; + } else { + $reason = "aborted by signal=$sigName"; + } + if ( $Pid == $$ ) { + # + # Parent logs a message + # + print(LOG $bpc->timeStamp, + "Aborting backup up after signal $sigName\n"); + + # + # Tell xfer to abort + # + $xfer->abort($reason); + + # + # Send ALRMs to BackupPC_tarExtract if we are using it + # + if ( $tarPid > 0 ) { + kill($bpc->sigName2num("ARLM"), $tarPid); + } + + # + # Schedule a 20 second timer in case the clean + # abort doesn't complete + # + alarm(20); + } else { + # + # Children ignore anything other than ALRM and INT + # + if ( $sigName ne "ALRM" && $sigName ne "INT" ) { + return; + } + + # + # The child also tells xfer to abort + # + $xfer->abort($reason); + + # + # Schedule a 15 second timer in case the clean + # abort doesn't complete + # + alarm(15); + } + $SigName = $sigName; + $Abort = 1; + return; + } + + # + # This is a second signal: time to clean up. + # + if ( $Pid != $$ && ($sigName eq "ALRM" || $sigName eq "INT") ) { + # + # Children quit quietly on ALRM or INT + # + exit(1) + } # # Ignore other signals in children # return if ( $Pid != $$ ); - print(LOG $bpc->timeStamp, "cleaning up after signal $signame\n"); - $SIG{$signame} = 'IGNORE'; + $SIG{$sigName} = 'IGNORE'; UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); - $XferLOG->write(\"exiting after signal $signame\n"); - $XferLOG->close(); + $XferLOG->write(\"exiting after signal $sigName\n"); if ( @xferPid ) { - kill(2, @xferPid); + kill($bpc->sigName2num("INT"), @xferPid); sleep(1); - kill(9, @xferPid); + kill($bpc->sigName2num("KILL"), @xferPid); } if ( $tarPid > 0 ) { - kill(2, $tarPid); + kill($bpc->sigName2num("INT"), $tarPid); sleep(1); - kill(9, $tarPid); - } - unlink("$Dir/timeStamp.level0"); - unlink("$Dir/NewFileList"); - unlink("$Dir/XferLOG.bad"); - unlink("$Dir/XferLOG.bad$fileExt"); - rename("$Dir/XferLOG$fileExt", "$Dir/XferLOG.bad$fileExt"); - $bpc->RmTreeDefer("$TopDir/trash", "$Dir/new") if ( -d "$Dir/new" ); - if ( $signame eq "INT" ) { - print("dump failed: aborted by user (signal=$signame)\n"); + kill($bpc->sigName2num("KILL"), $tarPid); + } + if ( $sigName eq "INT" ) { + $stat{hostError} = "aborted by user (signal=$sigName)"; } else { - print("dump failed: received signal=$signame\n"); + $stat{hostError} = "received signal=$sigName"; + } + BackupFailCleanup(); +} + +sub CheckForNewFiles +{ + if ( -f _ && $File::Find::name !~ /\/fattrib$/ ) { + $nFilesTotal++; + } elsif ( -d _ ) { + # + # No need to check entire tree + # + $File::Find::prune = 1 if ( $nFilesTotal ); + } +} + +sub BackupFailCleanup +{ + my $fileExt = $Conf{CompressLevel} > 0 ? ".z" : ""; + my $keepPartial = 0; + + # + # We keep this backup if it is a full and we actually backed + # up some files. + # + if ( $type eq "full" ) { + if ( $nFilesTotal == 0 && $xfer->getStats->{fileCnt} == 0 ) { + # + # Xfer didn't report any files, but check in the new + # directory just in case. + # + find(\&CheckForNewFiles, "$Dir/new"); + $keepPartial = 1 if ( $nFilesTotal ); + } else { + # + # Xfer reported some files + # + $keepPartial = 1; + } } + + # + # Don't keep partials if they are disabled + # + $keepPartial = 0 if ( $Conf{PartialAgeMax} < 0 ); + + if ( !$keepPartial ) { + # + # No point in saving this dump; get rid of eveything. + # + $XferLOG->close(); + unlink("$Dir/timeStamp.level0") if ( -f "$Dir/timeStamp.level0" ); + unlink("$Dir/SmbLOG.bad") if ( -f "$Dir/SmbLOG.bad" ); + unlink("$Dir/SmbLOG.bad$fileExt") if ( -f "$Dir/SmbLOG.bad$fileExt" ); + unlink("$Dir/XferLOG.bad") if ( -f "$Dir/XferLOG.bad" ); + unlink("$Dir/XferLOG.bad$fileExt") if ( -f "$Dir/XferLOG.bad$fileExt" ); + unlink("$Dir/NewFileList") if ( -f "$Dir/NewFileList" ); + rename("$Dir/XferLOG$fileExt", "$Dir/XferLOG.bad$fileExt"); + $bpc->RmTreeDefer("$TopDir/trash", "$Dir/new") if ( -d "$Dir/new" ); + print("dump failed: $stat{hostError}\n"); + $XferLOG->close(); + print("link $clientURI\n") if ( $needLink ); + exit(1); + } + # + # Ok, now we should save this as a partial dump + # + $type = "partial"; + my $newNum = BackupSave(); + print("dump failed: $stat{hostError}\n"); print("link $clientURI\n") if ( $needLink ); - exit(1); + print(LOG $bpc->timeStamp, "Saved partial dump $newNum\n"); + exit(2); } # @@ -870,12 +1080,21 @@ sub BackupExpire my($client) = @_; my($Dir) = "$TopDir/pc/$client"; my(@Backups) = $bpc->BackupInfoRead($client); - my($cntFull, $cntIncr, $firstFull, $firstIncr, $oldestIncr, $oldestFull); + my($cntFull, $cntIncr, $firstFull, $firstIncr, $oldestIncr, + $oldestFull, $changes); + if ( $Conf{FullKeepCnt} <= 0 ) { + print(LOG $bpc->timeStamp, + "Invalid value for \$Conf{FullKeepCnt}=$Conf{FullKeepCnt}\n"); + print(STDERR + "Invalid value for \$Conf{FullKeepCnt}=$Conf{FullKeepCnt}\n") + if ( $opts{v} ); + return; + } while ( 1 ) { $cntFull = $cntIncr = 0; $oldestIncr = $oldestFull = 0; - for ( $i = 0 ; $i < @Backups ; $i++ ) { + for ( my $i = 0 ; $i < @Backups ; $i++ ) { if ( $Backups[$i]{type} eq "full" ) { $firstFull = $i if ( $cntFull == 0 ); $cntFull++; @@ -903,20 +1122,35 @@ sub BackupExpire # print(LOG $bpc->timeStamp, "removing incr backup $Backups[$firstIncr]{num}\n"); - $bpc->RmTreeDefer("$TopDir/trash", - "$Dir/$Backups[$firstIncr]{num}"); - unlink("$Dir/SmbLOG.$Backups[$firstIncr]{num}") - if ( -f "$Dir/SmbLOG.$Backups[$firstIncr]{num}" ); - unlink("$Dir/SmbLOG.$Backups[$firstIncr]{num}.z") - if ( -f "$Dir/SmbLOG.$Backups[$firstIncr]{num}.z" ); - unlink("$Dir/XferLOG.$Backups[$firstIncr]{num}") - if ( -f "$Dir/XferLOG.$Backups[$firstIncr]{num}" ); - unlink("$Dir/XferLOG.$Backups[$firstIncr]{num}.z") - if ( -f "$Dir/XferLOG.$Backups[$firstIncr]{num}.z" ); - splice(@Backups, $firstIncr, 1); - } elsif ( ($cntFull > $Conf{FullKeepCnt} - || ($cntFull > $Conf{FullKeepCntMin} - && $oldestFull > $Conf{FullAgeMax})) + BackupRemove($client, \@Backups, $firstIncr); + $changes++; + next; + } + + # + # Delete any old full backups, according to $Conf{FullKeepCntMin} + # and $Conf{FullAgeMax}. + # + # First make sure that $Conf{FullAgeMax} is at least bigger + # than $Conf{FullPeriod} * $Conf{FullKeepCnt}, including + # the exponential array case. + # + my $fullKeepCnt = $Conf{FullKeepCnt}; + $fullKeepCnt = [$fullKeepCnt] if ( ref($fullKeepCnt) ne "ARRAY" ); + my $fullAgeMax; + my $fullPeriod = int(0.5 + $Conf{FullPeriod}); + $fullPeriod = 7 if ( $fullPeriod <= 0 ); + for ( my $i = 0 ; $i < @$fullKeepCnt ; $i++ ) { + $fullAgeMax += $fullKeepCnt->[$i] * $fullPeriod; + $fullPeriod *= 2; + } + $fullAgeMax += $fullPeriod; # add some buffer + + if ( $cntFull > $Conf{FullKeepCntMin} + && $oldestFull > $Conf{FullAgeMax} + && $oldestFull > $fullAgeMax + && $Conf{FullKeepCntMin} > 0 + && $Conf{FullAgeMax} > 0 && (@Backups <= $firstFull + 1 || !$Backups[$firstFull + 1]{noFill}) ) { # @@ -927,23 +1161,266 @@ sub BackupExpire # filled.) # print(LOG $bpc->timeStamp, - "removing full backup $Backups[$firstFull]{num}\n"); - $bpc->RmTreeDefer("$TopDir/trash", - "$Dir/$Backups[$firstFull]{num}"); - unlink("$Dir/SmbLOG.$Backups[$firstFull]{num}") - if ( -f "$Dir/SmbLOG.$Backups[$firstFull]{num}" ); - unlink("$Dir/SmbLOG.$Backups[$firstFull]{num}.z") - if ( -f "$Dir/SmbLOG.$Backups[$firstFull]{num}.z" ); - unlink("$Dir/XferLOG.$Backups[$firstFull]{num}") - if ( -f "$Dir/XferLOG.$Backups[$firstFull]{num}" ); - unlink("$Dir/XferLOG.$Backups[$firstFull]{num}.z") - if ( -f "$Dir/XferLOG.$Backups[$firstFull]{num}.z" ); - splice(@Backups, $firstFull, 1); - } else { + "removing old full backup $Backups[$firstFull]{num}\n"); + BackupRemove($client, \@Backups, $firstFull); + $changes++; + next; + } + + # + # Do new-style full backup expiry, which includes the the case + # where $Conf{FullKeepCnt} is an array. + # + last if ( !BackupFullExpire($client, \@Backups) ); + } + $bpc->BackupInfoWrite($client, @Backups) if ( $changes ); +} + +# +# Handle full backup expiry, using exponential periods. +# +sub BackupFullExpire +{ + my($client, $Backups) = @_; + my $fullCnt = 0; + my $fullPeriod = $Conf{FullPeriod}; + my $origFullPeriod = $fullPeriod; + my $fullKeepCnt = $Conf{FullKeepCnt}; + my $fullKeepIdx = 0; + my(@delete, @fullList); + + # + # Don't delete anything if $Conf{FullPeriod} or $Conf{FullKeepCnt} are + # not defined - possibly a corrupted config.pl file. + # + return if ( !defined($Conf{FullPeriod}) || !defined($Conf{FullKeepCnt}) ); + + # + # If regular backups are still disabled with $Conf{FullPeriod} < 0, + # we still expire backups based on a typical FullPeriod value - weekly. + # + $fullPeriod = 7 if ( $fullPeriod <= 0 ); + + $fullKeepCnt = [$fullKeepCnt] if ( ref($fullKeepCnt) ne "ARRAY" ); + + for ( my $i = 0 ; $i < @$Backups ; $i++ ) { + next if ( $Backups->[$i]{type} ne "full" ); + push(@fullList, $i); + } + for ( my $k = @fullList - 1 ; $k >= 0 ; $k-- ) { + my $i = $fullList[$k]; + my $prevFull = $fullList[$k-1] if ( $k > 0 ); + # + # Don't delete any full that is followed by an unfilled backup, + # since it is needed for restore. + # + my $noDelete = $i + 1 < @$Backups ? $Backups->[$i+1]{noFill} : 0; + + if ( !$noDelete && + ($fullKeepIdx >= @$fullKeepCnt + || $k > 0 + && $fullKeepIdx > 0 + && $Backups->[$i]{startTime} - $Backups->[$prevFull]{startTime} + < ($fullPeriod - $origFullPeriod / 2) * 24 * 3600 + ) + ) { + # + # Delete the full backup + # + #print("Deleting backup $i ($prevFull)\n"); + unshift(@delete, $i); + } else { + $fullCnt++; + while ( $fullKeepIdx < @$fullKeepCnt + && $fullCnt >= $fullKeepCnt->[$fullKeepIdx] ) { + $fullKeepIdx++; + $fullCnt = 0; + $fullPeriod = 2 * $fullPeriod; + } + } + } + # + # Now actually delete the backups + # + for ( my $i = @delete - 1 ; $i >= 0 ; $i-- ) { + print(LOG $bpc->timeStamp, + "removing full backup $Backups->[$delete[$i]]{num}\n"); + BackupRemove($client, $Backups, $delete[$i]); + } + return @delete; +} + +# +# Removes any partial backups +# +sub BackupPartialRemove +{ + my($client, $Backups) = @_; + + for ( my $i = @$Backups - 1 ; $i >= 0 ; $i-- ) { + next if ( $Backups->[$i]{type} ne "partial" ); + BackupRemove($client, $Backups, $i); + } +} + +sub BackupSave +{ + my @Backups = $bpc->BackupInfoRead($client); + my $num = -1; + my $newFilesFH; + + # + # Since we got a good backup we should remove any partial dumps + # (the new backup might also be a partial, but that's ok). + # + BackupPartialRemove($client, \@Backups); + + # + # Number the new backup + # + for ( my $i = 0 ; $i < @Backups ; $i++ ) { + $num = $Backups[$i]{num} if ( $num < $Backups[$i]{num} ); + } + $num++; + $bpc->RmTreeDefer("$TopDir/trash", "$Dir/$num") if ( -d "$Dir/$num" ); + if ( !rename("$Dir/new", "$Dir/$num") ) { + print(LOG $bpc->timeStamp, "Rename $Dir/new -> $Dir/$num failed\n"); + $stat{xferOK} = 0; + } + $needLink = 1 if ( -f "$Dir/NewFileList" ); + + # + # Add the new backup information to the backup file + # + my $i = @Backups; + $Backups[$i]{num} = $num; + $Backups[$i]{type} = $type; + $Backups[$i]{startTime} = $startTime; + $Backups[$i]{endTime} = $endTime; + $Backups[$i]{size} = $sizeTotal; + $Backups[$i]{nFiles} = $nFilesTotal; + $Backups[$i]{xferErrs} = $stat{xferErrCnt} || 0; + $Backups[$i]{xferBadFile} = $stat{xferBadFileCnt} || 0; + $Backups[$i]{xferBadShare} = $stat{xferBadShareCnt} || 0; + $Backups[$i]{nFilesExist} = $nFilesExist; + $Backups[$i]{sizeExist} = $sizeExist; + $Backups[$i]{sizeExistComp} = $sizeExistComp; + $Backups[$i]{tarErrs} = $tarErrs; + $Backups[$i]{compress} = $Conf{CompressLevel}; + $Backups[$i]{noFill} = $type eq "incr" ? 1 : 0; + $Backups[$i]{level} = $type eq "incr" ? 1 : 0; + $Backups[$i]{mangle} = 1; # name mangling always on for v1.04+ + $Backups[$i]{xferMethod} = $Conf{XferMethod}; + $Backups[$i]{charset} = $Conf{ClientCharset}; + # + # Save the main backups file + # + $bpc->BackupInfoWrite($client, @Backups); + # + # Save just this backup's info in case the main backups file + # gets corrupted + # + BackupPC::Storage->backupInfoWrite($Dir, $Backups[$i]{num}, + $Backups[$i]); + + unlink("$Dir/timeStamp.level0") if ( -f "$Dir/timeStamp.level0" ); + foreach my $ext ( qw(bad bad.z) ) { + next if ( !-f "$Dir/XferLOG.$ext" ); + unlink("$Dir/XferLOG.$ext.old") if ( -f "$Dir/XferLOG.$ext" ); + rename("$Dir/XferLOG.$ext", "$Dir/XferLOG.$ext.old"); + } + + # + # Now remove the bad files, replacing them if possible with links to + # earlier backups. + # + foreach my $f ( $xfer->getBadFiles ) { + my $j; + my $shareM = $bpc->fileNameEltMangle($f->{share}); + my $fileM = $bpc->fileNameMangle($f->{file}); + unlink("$Dir/$num/$shareM/$fileM"); + for ( $j = $i - 1 ; $j >= 0 ; $j-- ) { + my $file; + if ( $Backups[$j]{mangle} ) { + $file = "$shareM/$fileM"; + } else { + $file = "$f->{share}/$f->{file}"; + } + next if ( !-f "$Dir/$Backups[$j]{num}/$file" ); + + my($exists, $digest, $origSize, $outSize, $errs) + = BackupPC::PoolWrite::LinkOrCopy( + $bpc, + "$Dir/$Backups[$j]{num}/$file", + $Backups[$j]{compress}, + "$Dir/$num/$shareM/$fileM", + $Conf{CompressLevel}); + if ( !$exists ) { + # + # the hard link failed, most likely because the target + # file has too many links. We have copied the file + # instead, so add this to the new file list. + # + if ( !defined($newFilesFH) ) { + my $str = "Appending to NewFileList for $shareM/$fileM\n"; + $XferLOG->write(\$str); + open($newFilesFH, ">>", "$TopDir/pc/$client/NewFileList") + || die("can't open $TopDir/pc/$client/NewFileList"); + binmode($newFilesFH); + } + if ( -f "$Dir/$num/$shareM/$fileM" ) { + print($newFilesFH "$digest $origSize $shareM/$fileM\n"); + } else { + my $str = "Unable to link/copy $num/$f->{share}/$f->{file}" + . " to $Backups[$j]{num}/$f->{share}/$f->{file}\n"; + $XferLOG->write(\$str); + } + } else { + my $str = "Bad file $num/$f->{share}/$f->{file} replaced" + . " by link to" + . " $Backups[$j]{num}/$f->{share}/$f->{file}\n"; + $XferLOG->write(\$str); + } last; } + if ( $j < 0 ) { + my $str = "Removed bad file $num/$f->{share}/$f->{file}" + . " (no older copy to link to)\n"; + $XferLOG->write(\$str); + } } - $bpc->BackupInfoWrite($client, @Backups); + close($newFilesFH) if ( defined($newFilesFH) ); + $XferLOG->close(); + rename("$Dir/XferLOG$fileExt", "$Dir/XferLOG.$num$fileExt"); + rename("$Dir/NewFileList", "$Dir/NewFileList.$num"); + + return $num; +} + +# +# Removes a specific backup +# +sub BackupRemove +{ + my($client, $Backups, $idx) = @_; + my($Dir) = "$TopDir/pc/$client"; + + if ( $Backups->[$idx]{num} eq "" ) { + print("BackupRemove: ignoring empty backup number for idx $idx\n"); + return; + } + + $bpc->RmTreeDefer("$TopDir/trash", + "$Dir/$Backups->[$idx]{num}"); + unlink("$Dir/SmbLOG.$Backups->[$idx]{num}") + if ( -f "$Dir/SmbLOG.$Backups->[$idx]{num}" ); + unlink("$Dir/SmbLOG.$Backups->[$idx]{num}.z") + if ( -f "$Dir/SmbLOG.$Backups->[$idx]{num}.z" ); + unlink("$Dir/XferLOG.$Backups->[$idx]{num}") + if ( -f "$Dir/XferLOG.$Backups->[$idx]{num}" ); + unlink("$Dir/XferLOG.$Backups->[$idx]{num}.z") + if ( -f "$Dir/XferLOG.$Backups->[$idx]{num}.z" ); + splice(@{$Backups}, $idx, 1); } sub CorrectHostCheck @@ -979,9 +1456,9 @@ sub pidHandler # sub UserCommandRun { - my($type) = @_; + my($cmdType, $sharename) = @_; - return if ( !defined($Conf{$type}) ); + return if ( !defined($Conf{$cmdType}) ); my $vars = { xfer => $xfer, client => $client, @@ -997,10 +1474,17 @@ sub UserCommandRun XferLOG => $XferLOG, stat => \%stat, xferOK => $stat{xferOK} || 0, + hostError => $stat{hostError}, type => $type, + cmdType => $cmdType, }; - my $cmd = $bpc->cmdVarSubstitute($Conf{$type}, $vars); - $XferLOG->write(\"Executing $type: @$cmd\n"); + + if ($cmdType eq 'DumpPreShareCmd' || $cmdType eq 'DumpPostShareCmd') { + $vars->{share} = $sharename; + } + + my $cmd = $bpc->cmdVarSubstitute($Conf{$cmdType}, $vars); + $XferLOG->write(\"Executing $cmdType: @$cmd\n"); # # Run the user's command, dumping the stdout/stderr into the # Xfer log file. Also supply the optional $vars and %Conf in