From 617af75f7419e95a9c3ea05b05cf21957acc331c Mon Sep 17 00:00:00 2001 From: cbarratt Date: Wed, 28 Jun 2006 07:34:37 +0000 Subject: [PATCH 1/1] * Added multi-level incrementals. Still needs testing. * Decoupled BackupPC_nightly from BackupPC_dump * Various other changes --- ChangeLog | 70 +++++++++-- bin/BackupPC | 20 ++- bin/BackupPC_archive | 12 +- bin/BackupPC_dump | 215 +++++++++++++++++++++++--------- bin/BackupPC_fixupBackupSummary | 42 +++---- bin/BackupPC_nightly | 102 +++++++-------- bin/BackupPC_restore | 12 +- bin/BackupPC_sendEmail | 5 + bin/BackupPC_tarCreate | 6 +- bin/BackupPC_zipCreate | 7 +- cgi-bin/BackupPC_Admin | 2 +- conf/config.pl | 142 ++++++++++++++++++--- doc-src/BackupPC.pod | 52 +++++--- init.d/README | 14 ++- lib/BackupPC/CGI/EditConfig.pm | 5 +- lib/BackupPC/CGI/HostInfo.pm | 2 + lib/BackupPC/CGI/LOGlist.pm | 46 +++++-- lib/BackupPC/CGI/View.pm | 25 ++-- lib/BackupPC/Config/Meta.pm | 6 + lib/BackupPC/Lang/de.pm | 17 +++ lib/BackupPC/Lang/en.pm | 17 +++ lib/BackupPC/Lang/es.pm | 17 +++ lib/BackupPC/Lang/fr.pm | 17 +++ lib/BackupPC/Lang/it.pm | 29 ++++- lib/BackupPC/Lang/nl.pm | 17 +++ lib/BackupPC/Lang/pt_br.pm | 17 +++ lib/BackupPC/Lib.pm | 62 ++++++++- lib/BackupPC/PoolWrite.pm | 79 ++++++++++-- lib/BackupPC/Xfer/BackupPCd.pm | 5 +- lib/BackupPC/Xfer/Rsync.pm | 7 +- lib/BackupPC/Xfer/Smb.pm | 9 +- lib/BackupPC/Xfer/Tar.pm | 5 +- makeDist | 1 + 33 files changed, 846 insertions(+), 238 deletions(-) diff --git a/ChangeLog b/ChangeLog index f8725f5..a82518d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,13 +21,49 @@ # Version __VERSION__, __RELEASEDATE__ #------------------------------------------------------------------------ -* Added config and host CGI editor. +* Added configuration and host CGI editor. -* Added rsync hardlink support. +* Added rsync hardlink support. Requires latest version of + File::RsyncP. + +* Decoupled BackupPC_dump from BackupPC_nightly by making + asynchronous file linking/delete robust to race conditions. + Now only BackupPC_nightly and BackupPC_link are mutually + exclusive so only one runs at a time, and BackupPC_dump and + BackupPC_restore can run anytime. + +* Added support for multi-level incrementals. In the style of dump(1), + the level of each incremental can be specified. Each incremental + backups up everything since the most recent backup of a lower level + (fulls are always level 0). Previous behavior was all incrementals + were level 1, meaning they backed up everything since the last full + (level 0). Default configuration is all incrementals are level 1. * Server file names are now in utf8 and optional conversion to/from client name charsets is done. +* Backup metadata is now additionally saved to pc/HOST/nnn/backupInfo, + in addition to pc/HOST/backups. In case pc/HOST/backups gets trashed, + then a new script BackupPC_fixupBackupSummary can read the per-backup + metadata from pc/HOST/nnn/backupInfo and reconstruct the backups file. + Roberto Moreno also pointed out an early error in the CVS version. + +* In conf/config.pl, changed --devices to -D in $Conf{RsyncArgs} + and $Conf{RsyncRestoreArgs} to fix "fileListReceive failed" and + "Can't open .../f%2f for empty output" errors with rsync 2.6.7+. + Fix proposed by Justin Pessa and Vincent Ho, and confirmed by + Dan Niles. + +* Added Storage module and Storage::Text which localizes all the + text data file reading/writing (eg: backups, restores, archives + and config.pl files). Added read verify after all write + operations for robustness. Additional backends (eg: SQL) + can be added in the future as new subclasses of the Storage + module. + +* Added Config module, and Config::Meta that contains meta data + about configuration parameters. + * Added Slackware init.d script from Tony Nelson. * Fixed error reporting when restore/archive fail to write the @@ -35,16 +71,17 @@ * Applied patch from Marc Prewitt for DumpPreShareCmd and DumpPostShareCmd. +* Added checking of exit status of Dump/Restore/Archive Pre/Post UserCmd, + requested by Kiko Jover, Matthias Bertschy and others. + * Apply patch from Pete Wenzel to add smbClientPath => $Conf{SmbClientPath} to DumpPreUserCmd etc. * Added Portuguese Brazillian pt_br.pm from Reginaldo Ferreira. -* Jean-Michel Beuken reported several bugs in CVS 3.0.0. +* Jean-Michel Beuken reported several bugs in configure.pl in CVS 3.0.0. -* Applied Lorenzo Cappelletti's it.pm patch. - -* Applied Wander Winkelhorst's nl.pm patch. +* Old backup email warnings now ignore partials requested by Samuel Bancal * Applied patch to bin/BackupPC_sendEmail from Marc Prewitt that ignores any file starting with "." in the pc directory when @@ -52,6 +89,12 @@ * Applied patch from Marc Prewitt to fix host queue order. +* Applied Lorenzo Cappelletti's it.pm patch. + +* Applied Wander Winkelhorst's nl.pm patch. + +* Applied Alberto Marconi's it.pm patch. + * Add NT_STATUS_FILE_LOCK_CONFLICT to pst read error check in BackupPC_sendEmail to fix bug reported by Dale Renton. @@ -64,7 +107,15 @@ * Changed ping output parsing to pick out average rtt time, based on patch from Ron Bickers. -* Fixed minor documentation typos from Richard Ames, JP Vossen. +* Removed leading "./" and top-level "./" directory from + zip archives generated by BackupPC_zipCreate. Reported + by Josh (hecktarzuli). + +* BackupPC_tarCreate and BackupPC_zipCreate now allow "@" + in share names. Reported by Robert Waldner. + +* NT_STATUS_INSUFF_SERVER_RESOURCES is now a fatal error for + smbclient transfers, suggested by Brian Shand. * Changed bin/BackupPC_archiveHost to use /bin/csh instead of /bin/sh. That way any errors in the pipeline are reported @@ -75,6 +126,11 @@ * Made shareName argument regexp checking more general to allow parens. +* Added some debian init.d instructions to init.d/README from + Bob de Wildt. + +* Documentation updates from Richard Ames, JP Vossen, Torsten Finke. + #------------------------------------------------------------------------ # Version 2.1.2pl1, __RELEASEDATE__ #------------------------------------------------------------------------ diff --git a/bin/BackupPC b/bin/BackupPC index f4e4af7..426b59e 100755 --- a/bin/BackupPC +++ b/bin/BackupPC @@ -388,7 +388,7 @@ sub Main_TryToRun_nightly }); $CmdQueueOn{$bpc->trashJob} = 1; } - if ( keys(%Jobs) == $trashCleanRunning && $RunNightlyWhenIdle == 1 ) { + if ( $RunNightlyWhenIdle == 1 ) { # # Queue multiple nightly jobs based on the configuration @@ -606,7 +606,14 @@ sub Main_TryToRun_Bg_or_User_Queue } } - while ( $RunNightlyWhenIdle == 0 ) { + # + # Run background jobs anytime. Previously they were locked out + # when BackupPC_nightly was running or pending with this + # condition on the while loop: + # + # while ( $RunNightlyWhenIdle == 0 ) + # + while ( 1 ) { local(*FH); my(@args, $progName, $type); my $nJobs = keys(%Jobs); @@ -836,7 +843,8 @@ sub Main_Check_Timeout $Conf{CompressLevel}, 1); LogFileOpen(); # - # Remember to run nightly script after current jobs are done + # Remember to run the nightly script when the next CmdQueue + # job is done. # $RunNightlyWhenIdle = 1; } @@ -1067,7 +1075,8 @@ sub Main_Check_Job_Messages # # This means the last BackupPC_nightly is done with # the pool clean, so it's ok to start running regular - # backups again. + # backups again. But starting in 3.0 regular jobs + # are decoupled from BackupPC_nightly. # $RunNightlyWhenIdle = 0; } @@ -1097,6 +1106,9 @@ sub Main_Check_Job_Messages #print(LOG $bpc->timeStamp, "BackupPC_nightly done; now" # . " have $BackupPCNightlyJobs running\n"); if ( $BackupPCNightlyJobs <= 0 ) { + # + # Last BackupPC_nightly has finished + # $BackupPCNightlyJobs = 0; $RunNightlyWhenIdle = 0; $CmdJob = ""; diff --git a/bin/BackupPC_archive b/bin/BackupPC_archive index e9ed33b..e1e9900 100644 --- a/bin/BackupPC_archive +++ b/bin/BackupPC_archive @@ -157,6 +157,10 @@ local(*RH, *WH); # Run an optional pre-archive command # UserCommandRun("ArchivePreUserCmd"); +if ( $? && $Conf{UserCmdCheckStatus} ) { + $stat{hostError} = "ArchivePreUserCmd returned error status $?"; + exit(ArchiveCleanup($client)); +} $NeedPostCmd = 1; $xfer = BackupPC::Xfer::Archive->new($bpc); @@ -271,7 +275,13 @@ sub ArchiveCleanup # # Run an optional post-archive command # - UserCommandRun("ArchivePostUserCmd") if ( $NeedPostCmd ); + if ( $NeedPostCmd ) { + UserCommandRun("ArchivePostUserCmd"); + if ( $? && $Conf{UserCmdCheckStatus} ) { + $stat{hostError} = "RestorePreUserCmd returned error status $?"; + $stat{xferOK} = 0; + } + } rename("$Dir/ArchiveLOG$fileExt", "$Dir/ArchiveLOG.$lastNum$fileExt"); rename("$Dir/$reqFileName", "$Dir/ArchiveInfo.$lastNum"); diff --git a/bin/BackupPC_dump b/bin/BackupPC_dump index 72f992c..7ba9e3e 100755 --- a/bin/BackupPC_dump +++ b/bin/BackupPC_dump @@ -193,7 +193,30 @@ mkpath($Dir, 0, 0777) if ( !-d $Dir ); if ( !-f "$Dir/LOCK" ) { open(LOCK, ">", "$Dir/LOCK") && close(LOCK); } -open(LOG, ">>", "$Dir/LOG"); + +my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); +my $logPath = sprintf("$Dir/LOG.%02d%04d", $mon + 1, $year + 1900); + +if ( !-f $logPath ) { + # + # Compress and prune old log files + # + my $lastLog = $Conf{MaxOldPerPCLogFiles} - 1; + foreach my $file ( $bpc->sortedPCLogFiles($client) ) { + if ( $lastLog <= 0 ) { + unlink($file); + next; + } + next if ( $file =~ /\.z$/ || !$Conf{CompressLevel} ); + BackupPC::FileZIO->compressCopy($file, + "$file.z", + undef, + $Conf{CompressLevel}, 1); + $lastLog--; + } +} + +open(LOG, ">>", $logPath); select(LOG); $| = 1; select(STDOUT); # @@ -273,17 +296,25 @@ if ( $opts{d} ) { print("DHCP $hostIP $clientURI\n"); } -my($needLink, @Backups, $type, $lastBkupNum, $lastFullBkupNum); -my $lastFull = 0; -my $lastIncr = 0; +my($needLink, @Backups, $type); +my($incrBaseTime, $incrBaseBkupNum, $incrBaseLevel, $incrLevel); +my $lastFullTime = 0; +my $lastIncrTime = 0; my $partialIdx = -1; my $partialNum; my $lastPartial = 0; -if ( $Conf{FullPeriod} == -1 && !$opts{f} && !$opts{i} - || $Conf{FullPeriod} == -2 ) { +# +# Maintain backward compatibility with $Conf{FullPeriod} == -1 or -2 +# meaning disable backups +# +$Conf{BackupsDisable} = -$Conf{FullPeriod} + if ( !$Conf{BackupsDisable} && $Conf{FullPeriod} < 0 ); + +if ( $Conf{BackupsDisable} == 1 && !$opts{f} && !$opts{i} + || $Conf{BackupsDisable} == 2 ) { print(STDERR "Exiting because backups are disabled with" - . " \$Conf{FullPeriod} = $Conf{FullPeriod}\n") if ( $opts{v} ); + . " \$Conf{BackupsDisable} = $Conf{BackupsDisable}\n") if ( $opts{v} ); # # Tell BackupPC to ignore old failed backups on hosts that # have backups disabled. @@ -360,23 +391,29 @@ if ( !$opts{i} && !$opts{f} && $StatusHost{backoffTime} > time ) { # BackupExpire($client); +my(@lastIdxByLevel, $incrCntSinceFull); + # # Read Backup information, and find times of the most recent full and -# incremental backups +# incremental backups. Also figure out which backup we will use +# as a starting point for an incremental. # @Backups = $bpc->BackupInfoRead($client); +## @Backups = sort( { $a->{startTime} <=> $b->{startTime} }, @Backups); for ( my $i = 0 ; $i < @Backups ; $i++ ) { $needLink = 1 if ( $Backups[$i]{nFilesNew} eq "" || -f "$Dir/NewFileList.$Backups[$i]{num}" ); - $lastBkupNum = $Backups[$i]{num}; if ( $Backups[$i]{type} eq "full" ) { - if ( $lastFull < $Backups[$i]{startTime} ) { - $lastFull = $Backups[$i]{startTime}; - $lastFullBkupNum = $Backups[$i]{num}; + $incrCntSinceFull = 0; + $lastIdxByLevel[0] = $i; + if ( $lastFullTime < $Backups[$i]{startTime} ) { + $lastFullTime = $Backups[$i]{startTime}; } } elsif ( $Backups[$i]{type} eq "incr" ) { - $lastIncr = $Backups[$i]{startTime} - if ( $lastIncr < $Backups[$i]{startTime} ); + $incrCntSinceFull++; + $lastIdxByLevel[$Backups[$i]{level}] = $i; + $lastIncrTime = $Backups[$i]{startTime} + if ( $lastIncrTime < $Backups[$i]{startTime} ); } elsif ( $Backups[$i]{type} eq "partial" ) { $partialIdx = $i; $lastPartial = $Backups[$i]{startTime}; @@ -389,12 +426,37 @@ for ( my $i = 0 ; $i < @Backups ; $i++ ) { # if ( @Backups == 0 || $opts{f} - || (!$opts{i} && (time - $lastFull > $Conf{FullPeriod} * 24*3600 - && time - $lastIncr > $Conf{IncrPeriod} * 24*3600)) ) { + || (!$opts{i} && (time - $lastFullTime > $Conf{FullPeriod} * 24*3600 + && time - $lastIncrTime > $Conf{IncrPeriod} * 24*3600)) ) { $type = "full"; -} elsif ( $opts{i} || (time - $lastIncr > $Conf{IncrPeriod} * 24*3600 - && time - $lastFull > $Conf{IncrPeriod} * 24*3600) ) { +} elsif ( $opts{i} || (time - $lastIncrTime > $Conf{IncrPeriod} * 24*3600 + && time - $lastFullTime > $Conf{IncrPeriod} * 24*3600) ) { $type = "incr"; + # + # For an incremental backup, figure out which level we should + # do and the index of the reference backup, which is the most + # recent backup at any lower level. + # + @{$Conf{IncrLevels}} = [$Conf{IncrLevels}] + unless ref($Conf{IncrLevels}) eq "ARRAY"; + @{$Conf{IncrLevels}} = [1] if ( !@{$Conf{IncrLevels}} ); + $incrCntSinceFull = $incrCntSinceFull % @{$Conf{IncrLevels}}; + $incrLevel = $Conf{IncrLevels}[$incrCntSinceFull]; + for ( my $i = 0 ; $i < $incrLevel ; $i++ ) { + my $idx = $lastIdxByLevel[$i]; + next if ( !defined($idx) ); + if ( !defined($incrBaseTime) + || $Backups[$idx]{startTime} < $incrBaseTime ) { + $incrBaseBkupNum = $Backups[$idx]{num}; + $incrBaseLevel = $Backups[$idx]{level}; + $incrBaseTime = $Backups[$idx]{startTime}; + } + } + # + # Can't find any earlier lower-level backup! Shouldn't + # happen - just do full instead + # + $type = "full" if ( !defined($incrBaseBkupNum) || $incrLevel < 1 ); } else { NothingToDo($needLink); } @@ -510,6 +572,12 @@ $ShareNames = [ $ShareNames ] unless ref($ShareNames) eq "ARRAY"; # Run an optional pre-dump command # UserCommandRun("DumpPreUserCmd"); +if ( $? && $Conf{UserCmdCheckStatus} ) { + print(LOG $bpc->timeStamp, + "DumpPreUserCmd returned error status $?... exiting\n"); + print("dump failed: DumpPreUserCmd returned error status $?\n"); + exit(1); +} $NeedPostCmd = 1; # @@ -527,6 +595,13 @@ for my $shareName ( @$ShareNames ) { } UserCommandRun("DumpPreShareCmd", $shareName); + if ( $? && $Conf{UserCmdCheckStatus} ) { + print(LOG $bpc->timeStamp, + "DumpPreShareCmd returned error status $?... exiting\n"); + print("dump failed: DumpPreShareCmd returned error status $?\n"); + UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); + exit(1); + } if ( $Conf{XferMethod} eq "tar" ) { # @@ -635,25 +710,24 @@ for my $shareName ( @$ShareNames ) { # Run the transport program # $xfer->args({ - host => $host, - client => $client, - hostIP => $hostIP, - shareName => $shareName, - pipeRH => *RH, - pipeWH => *WH, - XferLOG => $XferLOG, - newFilesFH => $newFilesFH, - outDir => $Dir, - type => $type, - lastFull => $lastFull, - lastBkupNum => $lastBkupNum, - lastFullBkupNum => $lastFullBkupNum, - backups => \@Backups, - compress => $Conf{CompressLevel}, - XferMethod => $Conf{XferMethod}, - logLevel => $Conf{XferLogLevel}, - pidHandler => \&pidHandler, - partialNum => $partialNum, + host => $host, + client => $client, + hostIP => $hostIP, + shareName => $shareName, + pipeRH => *RH, + pipeWH => *WH, + XferLOG => $XferLOG, + newFilesFH => $newFilesFH, + outDir => $Dir, + type => $type, + incrBaseTime => $incrBaseTime, + incrBaseBkupNum => $incrBaseBkupNum, + backups => \@Backups, + compress => $Conf{CompressLevel}, + XferMethod => $Conf{XferMethod}, + logLevel => $Conf{XferLogLevel}, + pidHandler => \&pidHandler, + partialNum => $partialNum, }); if ( !defined($logMsg = $xfer->start()) ) { @@ -788,7 +862,14 @@ for my $shareName ( @$ShareNames ) { } } - UserCommandRun("DumpPostShareCmd", $shareName) if ( $NeedPostCmd ); + if ( $NeedPostCmd ) { + UserCommandRun("DumpPostShareCmd", $shareName); + if ( $? && $Conf{UserCmdCheckStatus} ) { + print(LOG $bpc->timeStamp, + "DumpPostShareCmd returned error status $?... exiting\n"); + $stat{hostError} = "DumpPostShareCmd returned error status $?"; + } + } $stat{xferOK} = 0 if ( $stat{hostError} || $stat{hostAbort} ); if ( !$stat{xferOK} ) { @@ -835,6 +916,12 @@ if ( $stat{xferOK} && (my $errMsg = CorrectHostCheck($hostIP, $host)) ) { } UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); +if ( $? && $Conf{UserCmdCheckStatus} ) { + print(LOG $bpc->timeStamp, + "DumpPostUserCmd returned error status $?... exiting\n"); + $stat{hostError} = "DumpPostUserCmd returned error status $?"; + $stat{xferOK} = 0; +} close($newFilesFH) if ( defined($newFilesFH) ); my $endTime = time(); @@ -1098,7 +1185,7 @@ sub BackupExpire if ( $Backups[$i]{type} eq "full" ) { $firstFull = $i if ( $cntFull == 0 ); $cntFull++; - } else { + } elsif ( $Backups[$i]{type} eq "incr" ) { $firstIncr = $i if ( $cntIncr == 0 ); $cntIncr++; } @@ -1107,23 +1194,39 @@ sub BackupExpire if ( $cntIncr > 0 ); $oldestFull = (time - $Backups[$firstFull]{startTime}) / (24 * 3600) if ( $cntFull > 0 ); - if ( $cntIncr > $Conf{IncrKeepCnt} - || ($cntIncr > $Conf{IncrKeepCntMin} - && $oldestIncr > $Conf{IncrAgeMax}) - && (@Backups <= $firstIncr + 1 - || $Backups[$firstIncr]{noFill} - || !$Backups[$firstIncr + 1]{noFill}) ) { + + # + # With multi-level incrementals, several of the following + # incrementals might depend upon this one, so we have to + # delete all of the them. Figure out if that is possible + # by counting the number of consecutive incrementals that + # are unfilled and have a level higher than this one. + # + my $cntIncrDel = 1; + my $earliestIncr = $oldestIncr; + + for ( my $i = $firstIncr + 1 ; $i < @Backups ; $i++ ) { + last if ( $Backups[$i]{level} <= $Backups[$firstIncr]{level} + || !$Backups[$i]{noFill} ); + $cntIncrDel++; + $earliestIncr = (time - $Backups[$i]{startTime}) / (24 * 3600); + } + + if ( $cntIncr >= $Conf{IncrKeepCnt} + $cntIncrDel + || ($cntIncr >= $Conf{IncrKeepCntMin} + $cntIncrDel + && $earliestIncr > $Conf{IncrAgeMax}) ) { # - # Only delete an incr backup if the Conf settings are satisfied. - # We also must make sure that either this backup is the most - # recent one, or it is not filled, or the next backup is filled. - # (We can't deleted a filled incr if the next backup is not - # filled.) + # Only delete an incr backup if the Conf settings are satisfied + # for all $cntIncrDel incrementals. Since BackupRemove() does + # a splice() we need to do the deletes in the reverse order. # - print(LOG $bpc->timeStamp, - "removing incr backup $Backups[$firstIncr]{num}\n"); - BackupRemove($client, \@Backups, $firstIncr); - $changes++; + for ( my $i = $firstIncr + $cntIncrDel - 1 ; + $i >= $firstIncr ; $i-- ) { + print(LOG $bpc->timeStamp, + "removing incr backup $Backups[$i]{num}\n"); + BackupRemove($client, \@Backups, $i); + $changes++; + } next; } @@ -1308,8 +1411,8 @@ sub BackupSave $Backups[$i]{tarErrs} = $tarErrs; $Backups[$i]{compress} = $Conf{CompressLevel}; $Backups[$i]{noFill} = $type eq "incr" ? 1 : 0; - $Backups[$i]{level} = $type eq "incr" ? 1 : 0; - $Backups[$i]{mangle} = 1; # name mangling always on for v1.04+ + $Backups[$i]{level} = $incrLevel; + $Backups[$i]{mangle} = 1; # name mangling always on for v1.04+ $Backups[$i]{xferMethod} = $Conf{XferMethod}; $Backups[$i]{charset} = $Conf{ClientCharset}; # diff --git a/bin/BackupPC_fixupBackupSummary b/bin/BackupPC_fixupBackupSummary index 537bbc1..c163b99 100755 --- a/bin/BackupPC_fixupBackupSummary +++ b/bin/BackupPC_fixupBackupSummary @@ -56,6 +56,17 @@ my $Hosts = $bpc->HostInfoRead(); my @hostList; our(%backupInfo); +my %opts; + +if ( !getopts("l", \%opts) ) { + print STDERR <BackupInfoRead($host); - # - # Temporary: create backupInfo files in each backup - # directory - # - foreach ( my $i = 0 ; $i < @Backups ; $i++ ) { - BackupPC::Storage->backupInfoWrite($dir, $Backups[$i]{num}, - $Backups[$i]); - if ( 0 ) { - my $bkupNum = $Backups[$i]{num}; - if ( !-f "$dir/$bkupNum/backupInfo" ) { - my($dump) = Data::Dumper->new( - [ $Backups[$i]], - [qw(*backupInfo)]); - $dump->Indent(1); - if ( open(BKUPINFO, ">", "$dir/$bkupNum/backupInfo") ) { - print(BKUPINFO $dump->Dump); - close(BKUPINFO); - } - } - } - } - # # Look through the LOG files to get information about # completed backups. The data from the LOG file is @@ -121,7 +110,7 @@ foreach my $host ( @hostList ) { my @files = readdir(DIR); closedir(DIR); foreach my $file ( @files ) { - if ( $file =~ /^LOG(.\d+\.z)?/ ) { + if ( $opts{l} && $file =~ /^LOG(.\d+\.z)?/ ) { push(@LogFiles, $file); } elsif ( $file =~ /^(\d+)$/ ) { my $bkupNum = $1; @@ -154,6 +143,7 @@ foreach my $host ( @hostList ) { # @LogFiles = sort({-M "$dir/$a" <=> -M "$dir/$b"} @LogFiles); my $startTime; + my $fillFromNum; foreach my $file ( @LogFiles ) { my $f = BackupPC::FileZIO->open("$dir/$file", 0, $file =~ /\.z/); @@ -197,14 +187,12 @@ foreach my $host ( @hostList ) { noFill => $type eq "incr" ? 1 : 0, level => $type eq "incr" ? 1 : 0, mangle => 1, - noFill => $noFill; - fillFromNum => $fillFromNum; + fillFromNum => $fillFromNum, }; + $fillFromNum = $bkupNum if ( $type eq "full" ); } } - splice(@Backups, 2, 1); - # # Now merge any info from $BkupFromInfo and $BkupFromLOG # that is missing from @Backups. diff --git a/bin/BackupPC_nightly b/bin/BackupPC_nightly index a53e592..c38e68e 100755 --- a/bin/BackupPC_nightly +++ b/bin/BackupPC_nightly @@ -78,6 +78,14 @@ my $BinDir = $bpc->BinDir(); my %Conf = $bpc->Conf(); my(%Status, %Info, %Jobs, @BgQueue, @UserQueue, @CmdQueue); +# +# We delete unused pool files (link count 1) in sorted inode +# order by gathering batches. We delete the first half of +# each batch (ie: $PendingDeleteMax / 2 at a time). +# +my @PendingDelete; +my $PendingDeleteMax = 10240; + $bpc->ChildInit(); my %opts; @@ -107,13 +115,6 @@ if ( $opts{m} ) { eval($reply); } -########################################################################### -# When BackupPC_nightly starts, BackupPC will not run any simultaneous -# BackupPC_dump commands. We first do things that contend with -# BackupPC_dump, eg: aging per-PC log files etc. -########################################################################### -doPerPCLogFileAging() if ( $opts{m} ); - ########################################################################### # Get statistics on the pool, and remove files that have only one link. ########################################################################### @@ -202,11 +203,15 @@ for my $pool ( qw(pool cpool) ) { } } +sleep(10); +processPendingDeletes(1); + ########################################################################### # Tell BackupPC that it is now ok to start running BackupPC_dump # commands. We are guaranteed that no BackupPC_link commands will # run since only a single CmdQueue command runs at a time, and -# that means we are safe. +# that means we are safe. As of 3.x this is irrelevant since +# BackupPC_dump runs independent of BackupPC_dump. ########################################################################### printf("BackupPC_nightly lock_off\n"); @@ -219,42 +224,6 @@ if ( $opts{m} ) { doBackupInfoUpdate(); } -# -# Do per-PC log file aging -# -sub doPerPCLogFileAging -{ - my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); - if ( $mday == 1 ) { - foreach my $host ( keys(%Status) ) { - my $lastLog = $Conf{MaxOldPerPCLogFiles} - 1; - unlink("$TopDir/pc/$host/LOG.$lastLog") - if ( -f "$TopDir/pc/$host/LOG.$lastLog" ); - unlink("$TopDir/pc/$host/LOG.$lastLog.z") - if ( -f "$TopDir/pc/$host/LOG.$lastLog.z" ); - for ( my $i = $lastLog - 1 ; $i >= 0 ; $i-- ) { - my $j = $i + 1; - if ( -f "$TopDir/pc/$host/LOG.$i" ) { - rename("$TopDir/pc/$host/LOG.$i", - "$TopDir/pc/$host/LOG.$j"); - } elsif ( -f "$TopDir/pc/$host/LOG.$i.z" ) { - rename("$TopDir/pc/$host/LOG.$i.z", - "$TopDir/pc/$host/LOG.$j.z"); - } - } - # - # Compress the log file LOG -> LOG.0.z (if enabled). - # Otherwise, just rename LOG -> LOG.0. - # - BackupPC::FileZIO->compressCopy("$TopDir/pc/$host/LOG", - "$TopDir/pc/$host/LOG.0.z", - "$TopDir/pc/$host/LOG.0", - $Conf{CompressLevel}, 1); - open(LOG, ">", "$TopDir/pc/$host/LOG") && close(LOG); - } - } -} - # # Update the backupInfo files based on the backups file. # We do this just once a week (on Sun) since it is only @@ -282,7 +251,7 @@ sub doBackupInfoUpdate sub GetPoolStats { - my($nlinks, $nblocks) = (lstat($_))[3, 12]; + my($inode, $nlinks, $nblocks) = (lstat($_))[1, 3, 12]; if ( -d _ ) { $dirCnt++; @@ -293,7 +262,23 @@ sub GetPoolStats if ( $nlinks == 1 ) { $blkCntRm += $nblocks; $fileCntRm++; - unlink($_); + # + # Save the files for later batch deletion. + # + # This is so we can remove them in inode order, and additionally + # reduce any remaining chance of race condition of linking to + # pool files vs removing pool files. (Other aspects of the + # design should eliminate race conditions.) + # + my $fullPath = $File::Find::name; + push(@PendingDelete, { + inode => $inode, + path => $fullPath + } + ); + if ( @PendingDelete > $PendingDeleteMax ) { + processPendingDeletes(0); + } # # We must keep repeated files numbered sequential (ie: files # that have the same checksum are appended with _0, _1 etc). @@ -301,9 +286,8 @@ sub GetPoolStats # exists, or we remove any file of the form xxxx_nnn. We remember # the base name and fix it up later (not in the middle of find). # - my($baseName); - ($baseName = $File::Find::name) =~ s/_\d+$//; - $FixList{$baseName}++; + $fullPath =~ s/_\d+$//; + $FixList{$fullPath}++; } else { if ( /_(\d+)$/ ) { $fileRepMax = $1 + 1 if ( $fileRepMax <= $1 ); @@ -316,3 +300,23 @@ sub GetPoolStats $fileLinkTotal += $nlinks - 1; } } + +sub processPendingDeletes +{ + my($doAll) = @_; + my @delete; + + if ( !$doAll ) { + @delete = splice(@PendingDelete, 0, $PendingDeleteMax / 2); + } else { + @delete = @PendingDelete; + @PendingDelete = (); + } + for my $f ( sort({ $a->{inode} <=> $b->{inode} } @delete) ) { + my($nlinks) = (lstat($f->{path}))[3]; + + next if ( $nlinks != 1 ); + # print("Deleting $f->{path} ($f->{inode})\n"); + unlink($f->{path}); + } +} diff --git a/bin/BackupPC_restore b/bin/BackupPC_restore index 8f78316..ff9b413 100755 --- a/bin/BackupPC_restore +++ b/bin/BackupPC_restore @@ -208,6 +208,10 @@ local(*RH, *WH); # Run an optional pre-restore command # UserCommandRun("RestorePreUserCmd"); +if ( $? && $Conf{UserCmdCheckStatus} ) { + $stat{hostError} = "RestorePreUserCmd returned error status $?"; + exit(RestoreCleanup($client)); +} $NeedPostCmd = 1; if ( $Conf{XferMethod} eq "tar" ) { @@ -541,7 +545,13 @@ sub RestoreCleanup # # Run an optional post-restore command # - UserCommandRun("RestorePostUserCmd") if ( $NeedPostCmd ); + if ( $NeedPostCmd ) { + UserCommandRun("RestorePostUserCmd"); + if ( $? && $Conf{UserCmdCheckStatus} ) { + $stat{hostError} = "RestorePostUserCmd returned error status $?"; + $stat{xferOK} = 0; + } + } rename("$Dir/RestoreLOG$fileExt", "$Dir/RestoreLOG.$lastNum$fileExt"); rename("$Dir/$reqFileName", "$Dir/RestoreInfo.$lastNum"); diff --git a/bin/BackupPC_sendEmail b/bin/BackupPC_sendEmail index 8357bdc..3f82313 100755 --- a/bin/BackupPC_sendEmail +++ b/bin/BackupPC_sendEmail @@ -194,6 +194,11 @@ foreach my $host ( sort(keys(%Status)) ) { my $numBadOutlook = 0; for ( my $i = 0 ; $i < @Backups ; $i++ ) { my $fh; + # + # ignore partials -> only fulls and incrs should be used + # in figuring out when the last good backup was + # + next if ( $Backups[$i]{type} eq "partial" ); $lastNum = $Backups[$i]{num} if ( $lastNum < $Backups[$i]{num} ); if ( $Backups[$i]{type} eq "full" ) { $lastFull = $Backups[$i]{startTime} diff --git a/bin/BackupPC_tarCreate b/bin/BackupPC_tarCreate index 97b9878..f704255 100755 --- a/bin/BackupPC_tarCreate +++ b/bin/BackupPC_tarCreate @@ -96,7 +96,8 @@ EOF exit(1); } -if ( $opts{h} !~ /^([\w\.\s-]+)$/ ) { +if ( $opts{h} !~ /^([\w\.\s-]+)$/ + || $opts{h} =~ m{(^|/)\.\.(/|$)} ) { print(STDERR "$0: bad host name '$opts{h}'\n"); exit(1); } @@ -130,7 +131,8 @@ $Charset = $opts{e} if ( $opts{e} ne "" ); my $PathRemove = $1 if ( $opts{r} =~ /(.+)/ ); my $PathAdd = $1 if ( $opts{p} =~ /(.+)/ ); -if ( $opts{s} !~ /^([\w\s.\/$(){}[\]-]+)$/ && $opts{s} ne "*" ) { +if ( ($opts{s} !~ /^([\w\s.@\/$(){}[\]-]+)$/ + || $opts{s} =~ m{(^|/)\.\.(/|$)}) && $opts{s} ne "*" ) { print(STDERR "$0: bad share name '$opts{s}'\n"); exit(1); } diff --git a/bin/BackupPC_zipCreate b/bin/BackupPC_zipCreate index 51683fb..8e0b483 100755 --- a/bin/BackupPC_zipCreate +++ b/bin/BackupPC_zipCreate @@ -100,7 +100,8 @@ EOF exit(1); } -if ( $opts{h} !~ /^([\w\.\s-]+)$/ ) { +if ( $opts{h} !~ /^([\w\.\s-]+)$/ + || $opts{h} =~ m{(^|/)\.\.(/|$)} ) { print(STDERR "$0: bad host name '$opts{h}'\n"); exit(1); } @@ -141,7 +142,8 @@ $Charset = $opts{e} if ( $opts{e} ne "" ); my $PathRemove = $1 if ( $opts{r} =~ /(.+)/ ); my $PathAdd = $1 if ( $opts{p} =~ /(.+)/ ); -if ( $opts{s} !~ /^([\w\s.\/$(){}[\]-]+)$/ ) { +if ( $opts{s} !~ /^([\w\s.@\/$(){}[\]-]+)$/ + || $opts{s} =~ m{(^|/)\.\.(/|$)} ) { print(STDERR "$0: bad share name '$opts{s}'\n"); exit(1); } @@ -170,6 +172,7 @@ sub archiveWrite $ErrorCnt++; return; } + $dir = "/" if ( $dir eq "." ); $view->find($Num, $ShareName, $dir, 0, \&ZipWriteFile, $zipfh, $zipPathOverride); } diff --git a/cgi-bin/BackupPC_Admin b/cgi-bin/BackupPC_Admin index cd2e4a9..ac43d85 100755 --- a/cgi-bin/BackupPC_Admin +++ b/cgi-bin/BackupPC_Admin @@ -81,7 +81,7 @@ my %ActionDispatch = ( "Stop" => "StopServer", "adminOpts" => "AdminOptions", "editConfig" => "EditConfig", - #"editHosts" => "EditHosts", + "rss" => "RSS", ); # diff --git a/conf/config.pl b/conf/config.pl index e02fe27..824ae77 100644 --- a/conf/config.pl +++ b/conf/config.pl @@ -353,18 +353,6 @@ $Conf{ServerInitdStartCmd} = ''; # time taken for the backup, plus the granularity of $Conf{WakeupSchedule} # will make the actual backup interval a bit longer. # -# There are two special values for $Conf{FullPeriod}: -# -# -1 Don't do any regular backups on this machine. Manually -# requested backups (via the CGI interface) will still occur. -# -# -2 Don't do any backups on this machine. Manually requested -# backups (via the CGI interface) will be ignored. -# -# These special settings are useful for a client that is no longer -# being backed up (eg: a retired machine), but you wish to keep the -# last backups available for browsing or restoring to other machines. -# $Conf{FullPeriod} = 6.97; # @@ -481,6 +469,105 @@ $Conf{IncrKeepCnt} = 6; $Conf{IncrKeepCntMin} = 1; $Conf{IncrAgeMax} = 30; +# +# Level of each incremental. "Level" follows the terminology +# of dump(1). A full backup has level 0. A new incremental +# of level N will backup all files that have changed since +# the most recent backup of a lower level. +# +# The entries of $Conf{IncrLevels} apply in order to each +# incremental after each full backup. It wraps around until +# the next full backup. For example, these two settings +# have the same effect: +# +# $Conf{IncrLevels} = [1, 2, 3]; +# $Conf{IncrLevels} = [1, 2, 3, 1, 2, 3]; +# +# This means the 1st and 4th incrementals (level 1) go all +# the way back to the full. The 2nd and 3rd (and 5th and +# 6th) backups just go back to the immediate preceeding +# incremental. +# +# Specifying a sequence of multi-level incrementals will +# usually mean more than $Conf{IncrKeepCnt} incrementals will +# need to be kept, since lower level incrementals are needed +# to merge a complete view of a backup. For example, with +# +# $Conf{FullPeriod} = 7; +# $Conf{IncrPeriod} = 1; +# $Conf{IncrKeepCnt} = 6; +# $Conf{IncrLevels} = [1, 2, 3, 4, 5, 6]; +# +# there will be up to 11 incrementals in this case: +# +# backup #0 (full, level 0, oldest) +# backup #1 (incr, level 1) +# backup #2 (incr, level 2) +# backup #3 (incr, level 3) +# backup #4 (incr, level 4) +# backup #5 (incr, level 5) +# backup #6 (incr, level 6) +# backup #7 (full, level 0) +# backup #8 (incr, level 1) +# backup #9 (incr, level 2) +# backup #10 (incr, level 3) +# backup #11 (incr, level 4) +# backup #12 (incr, level 5, newest) +# +# Backup #1 (the oldest level 1 incremental) can't be deleted +# since backups 2..6 depend on it. Those 6 incrementals can't +# all be deleted since that would only leave 5 (#8..12). +# When the next incremental happens (level 6), the complete +# set of 6 older incrementals (#1..6) will be deleted, since +# that maintains the required number ($Conf{IncrKeepCnt}) +# of incrementals. This situation is reduced if you set +# shorter chains of multi-level incrementals, eg: +# +# $Conf{IncrLevels} = [1, 2, 3]; +# +# would only have up to 2 extra incremenals before all 3 +# are deleted. +# +# BackupPC as usual merges the full and the sequence +# of incrementals together so each incremental can be +# browsed and restored as though it is a complete backup. +# If you specify a long chain of incrementals then more +# backups need to be merged when browsing, restoring, +# or getting the starting point for rsync backups. +# In the example above (levels 1..6), browing backup +# #6 requires 7 different backups (#0..6) to be merged. +# +# Because of this merging and the additional incrementals +# that need to be kept, it is recommended that some +# level 1 incrementals be included in $Conf{IncrLevels}. +# +# Prior to version 3.0 incrementals were always level 1, +# meaning each incremental backed up all the files that +# changed since the last full. +# +$Conf{IncrLevels} = [1]; + +# +# Disable all full and incremental backups. These settings are +# useful for a client that is no longer being backed up +# (eg: a retired machine), but you wish to keep the last +# backups available for browsing or restoring to other machines. +# +# There are three values for $Conf{BackupsDisable}: +# +# 0 Backups are enabled. +# +# 1 Don't do any regular backups on this client. Manually +# requested backups (via the CGI interface) will still occur. +# +# 2 Don't do any backups on this client. Manually requested +# backups (via the CGI interface) will be ignored. +# +# In versions prior to 3.0 Backups were disabled by setting +# $Conf{FullPeriod} to -1 or -2. +# +$Conf{BackupsDisable} = 0; + # # A failed full backup is saved as a partial backup. The rsync # XferMethod can take advantage of the partial full when the next @@ -1116,7 +1203,7 @@ $Conf{RsyncArgs} = [ '--perms', '--owner', '--group', - '--devices', + '-D', '--links', '--times', '--block-size=2048', @@ -1150,7 +1237,7 @@ $Conf{RsyncRestoreArgs} = [ '--perms', '--owner', '--group', - '--devices', + '-D', '--links', '--times', '--block-size=2048', @@ -1536,6 +1623,29 @@ $Conf{RestorePostUserCmd} = undef; $Conf{ArchivePreUserCmd} = undef; $Conf{ArchivePostUserCmd} = undef; +# +# Whether the exit status of each PreUserCmd and +# PostUserCmd is checked. +# +# If set and the Dump/Restore/Archive Pre/Post UserCmd +# returns a non-zero exit status then the dump/restore/archive +# is aborted. To maintain backward compatibility (where +# the exit status in early versions was always ignored), +# this flag defaults to 0. +# +# If this flag is set and the Dump/Restore/Archive PreUserCmd +# fails then the matching Dump/Restore/Archive PostUserCmd is +# not executed. If DumpPreShareCmd returns a non-exit status, +# then DumpPostShareCmd is not executed, but the DumpPostUserCmd +# is still run (since DumpPreUserCmd must have previously +# succeeded). +# +# An example of a DumpPreUserCmd that might fail is a script +# that snapshots or dumps a database which fails because +# of some database error. +# +$Conf{UserCmdCheckStatus} = 0; + # # Override the client's host name. This allows multiple clients # to all refer to the same physical host. This should only be @@ -1879,8 +1989,9 @@ $Conf{CgiUserConfigEdit} = { IncrKeepCnt => 1, IncrKeepCntMin => 1, IncrAgeMax => 1, - PartialAgeMax => 1, + IncrLevels => 1, IncrFill => 1, + PartialAgeMax => 1, RestoreInfoKeepCnt => 1, ArchiveInfoKeepCnt => 1, BackupFilesOnly => 1, @@ -1934,6 +2045,7 @@ $Conf{CgiUserConfigEdit} = { ArchivePostUserCmd => 0, DumpPostShareCmd => 0, DumpPreShareCmd => 0, + UserCmdCheckStatus => 0, EMailNotifyMinDays => 1, EMailFromUserName => 1, EMailAdminUserName => 1, diff --git a/doc-src/BackupPC.pod b/doc-src/BackupPC.pod index c0b3b7d..cc4175e 100644 --- a/doc-src/BackupPC.pod +++ b/doc-src/BackupPC.pod @@ -251,18 +251,19 @@ Do not send subscription requests to this address! =item Other Programs of Interest If you want to mirror linux or unix files or directories to a remote server -you should consider rsync, L. BackupPC now uses +you should use rsync, L. BackupPC now uses rsync as a transport mechanism; if you are already an rsync user you can think of BackupPC as adding efficient storage (compression and pooling) and a convenient user interface to rsync. Unison is a utility that can do two-way, interactive, synchronization. -See L. +See L. An external wrapper around +rsync that maintains transfer data to enable two-way synchronization is +drsync; see L. -Three popular open source packages that do tape backup are -Amanda (L), -afbackup (L), and -Bacula (L). +Two popular open source packages that do tape backup are +Amanda (L) +and Bacula (L). Amanda can also backup WinXX machines to tape using samba. These packages can be used as back ends to BackupPC to backup the BackupPC server data to tape. @@ -276,7 +277,8 @@ and John Bowman's rlbackup (L). BackupPC provides many additional features, such as compressed storage, hardlinking any matching files (rather than just files with the same name), and storing special files without root privileges. But these other scripts -provide simple and effective solutions and are worthy of consideration. +provide simple and effective solutions and are definitely worthy of +consideration. =back @@ -567,6 +569,17 @@ sure the BackupPC user's group is chosen restrictively. On this installation, this is __BACKUPPCUSER__. +For security purposes you might choose to configre the BackupPC +user with the shell set to /bin/false. Since you might need to +run some BackupPC programs as the BackupPC user for testing +purposes, you can use the -s option to su to explicitly run +a shell, eg: + + su -s /bin/bash __BACKUPPCUSER__ + +Depending upon your configuration you might also need +the -l option. + =item Data Directory You need to decide where to put the data directory, below which @@ -1007,19 +1020,22 @@ it has started and all is ok. =head2 Step 7: Talking to BackupPC -Note: as of version 1.5.0, BackupPC no longer supports telnet -to its TCP port. First off, a unix domain socket is used -instead of a TCP port. (The TCP port can still be re-enabled -if your installation has apache and BackupPC running on different -machines.) Secondly, even if you still use the TCP port, the -messages exchanged over this interface are now protected by -an MD5 digest based on a shared secret (see $Conf{ServerMesgSecret}) -as well as sequence numbers and per-session unique keys, preventing -forgery and replay attacks. - You should verify that BackupPC is running by using BackupPC_serverMesg. This sends a message to BackupPC via the unix (or TCP) socket and prints -the response. +the response. Like all BackupPC programs, BackupPC_serverMesg +should be run as the BackupPC user (__BACKUPPCUSER__), so you +should + + su __BACKUPPCUSER__ + +before running BackupPC_serverMesg. If the BackupPC user is +configured with /bin/false as the shell, you can use the -s +option to su to explicitly run a shell, eg: + + su -s /bin/bash __BACKUPPCUSER__ + +Depending upon your configuration you might also need +the -l option. You can request status information and start and stop backups using this interface. This socket interface is mainly provided for the CGI interface diff --git a/init.d/README b/init.d/README index deee880..e08e77b 100644 --- a/init.d/README +++ b/init.d/README @@ -34,7 +34,19 @@ Debian Linux: When configure.pl is run, the script debian-backuppc is created. -(Can a Debian user add some instructions here??) +Copy the debian startup script: + + cp debian-backuppc /etc/init.d/backuppc + +Run the following command to install in rc.d: + + update-rc.d backuppc defaults + +Set the correct init.d rights: + + chmod 755 /etc/init.d/backuppc + +Usage: /etc/init.d/backuppc {start|stop|restart|reload} Suse Linux: ========== diff --git a/lib/BackupPC/CGI/EditConfig.pm b/lib/BackupPC/CGI/EditConfig.pm index c0c327d..da660af 100644 --- a/lib/BackupPC/CGI/EditConfig.pm +++ b/lib/BackupPC/CGI/EditConfig.pm @@ -10,7 +10,7 @@ # Craig Barratt # # COPYRIGHT -# Copyright (C) 2004 Craig Barratt +# Copyright (C) 2005 Craig Barratt # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -281,9 +281,11 @@ our %ConfigMenu = ( {name => "IncrKeepCnt"}, {name => "IncrKeepCntMin"}, {name => "IncrAgeMax"}, + {name => "IncrLevels"}, {name => "IncrFill"}, {text => "CfgEdit_Title_Blackouts"}, + {name => "BackupsDisable"}, {name => "BlackoutBadPingLimit"}, {name => "BlackoutGoodCnt"}, {name => "BlackoutPeriods"}, @@ -320,6 +322,7 @@ our %ConfigMenu = ( {name => "RestorePostUserCmd"}, {name => "ArchivePreUserCmd"}, {name => "ArchivePostUserCmd"}, + {name => "UserCmdCheckStatus"}, ], }, hosts => { diff --git a/lib/BackupPC/CGI/HostInfo.pm b/lib/BackupPC/CGI/HostInfo.pm index 5c20ef7..127b88f 100644 --- a/lib/BackupPC/CGI/HostInfo.pm +++ b/lib/BackupPC/CGI/HostInfo.pm @@ -148,6 +148,7 @@ EOF } my $age = sprintf("%.1f", (time - $Backups[$i]{startTime}) / (24*3600)); my $browseURL = "$MyURL?action=browse&host=${EscURI($host)}&num=$Backups[$i]{num}"; + my $level = $Backups[$i]{level}; my $filled = $Backups[$i]{noFill} ? $Lang->{No} : $Lang->{Yes}; $filled .= " ($Backups[$i]{fillFromNum}) " if ( $Backups[$i]{fillFromNum} ne "" ); @@ -156,6 +157,7 @@ EOF $Backups[$i]{num} $ltype $filled + $level $startTime $duration $age diff --git a/lib/BackupPC/CGI/LOGlist.pm b/lib/BackupPC/CGI/LOGlist.pm index 18cbdcd..f717a84 100644 --- a/lib/BackupPC/CGI/LOGlist.pm +++ b/lib/BackupPC/CGI/LOGlist.pm @@ -47,29 +47,24 @@ sub action ErrorExit($Lang->{Only_privileged_users_can_view_log_files}); } my $host = $In{host}; - my($url0, $hdr, $root, $str); + my($url0, $hdr, @files, $str); if ( $host ne "" ) { - $root = "$TopDir/pc/$host/LOG"; $url0 = "&host=${EscURI($host)}"; $hdr = "for host $host"; } else { - $root = "$LogDir/LOG"; $url0 = ""; $hdr = ""; } - for ( my $i = -1 ; ; $i++ ) { - my $url1 = ""; - my $file = $root; - if ( $i >= 0 ) { - $file .= ".$i"; - $url1 = "&num=$i"; - } - $file .= ".z" if ( !-f $file && -f "$file.z" ); - last if ( !-f $file ); + + foreach my $file ( $bpc->sortedPCLogFiles($host) ) { + my $url1 = "&num=$1" if ( $file =~ /LOG\.(\d+)(\.z)?$/ ); + $url1 = "&num=" if ( $file =~ /LOG(\.z)?$/ ); + next if ( !-f $file ); my $mtimeStr = $bpc->timeStamp((stat($file))[9], 1); my $size = (stat($file))[7]; + (my $fStr = $file) =~ s{.*/}{}; $str .= < $file + $fStr $size $mtimeStr EOF @@ -80,4 +75,29 @@ EOF Trailer(); } +sub compareLOGName +{ + #my($a, $b) = @_; + + my $na = $1 if ( $a =~ /LOG\.(\d+)/ ); + my $nb = $1 if ( $b =~ /LOG\.(\d+)/ ); + + if ( length($na) >= 5 && length($nb) >= 5 ) { + # + # Both new style. Bigger numbers are more recent. + # + return $nb <=> $na; + } elsif ( length($na) >= 5 && length($nb) < 5 ) { + return -1; + } elsif ( length($na) < 5 && length($nb) >= 5 ) { + return 1; + } else { + # + # Both old style. Smaller numbers are more recent. + # + return $na - $nb; + } +} + + 1; diff --git a/lib/BackupPC/CGI/View.pm b/lib/BackupPC/CGI/View.pm index ae03ba4..172e0e1 100644 --- a/lib/BackupPC/CGI/View.pm +++ b/lib/BackupPC/CGI/View.pm @@ -78,20 +78,22 @@ sub action } elsif ( $type eq "ArchiveErr" ) { $file = "$TopDir/pc/$host/ArchiveLOG$ext"; $comment = $Lang->{Extracting_only_Errors}; - } elsif ( $host ne "" && $type eq "config" ) { - $file = "$TopDir/pc/$host/config.pl"; - $file = "$TopDir/conf/$host.pl" - if ( $host ne "config" && -f "$TopDir/conf/$host.pl" - && !-f $file ); - } elsif ( $type eq "docs" ) { - $file = "$BinDir/../doc/BackupPC.html"; } elsif ( $type eq "config" ) { - $file = "$TopDir/conf/config.pl"; + # Note: only works for Storage::Text + $file = $bpc->{storage}->ConfigPath($host); } elsif ( $type eq "hosts" ) { - $file = "$TopDir/conf/hosts"; + # Note: only works for Storage::Text + $file = $bpc->ConfDir() . "/hosts"; $linkHosts = 1; + } elsif ( $type eq "docs" ) { + $file = "$BinDir/../doc/BackupPC.html"; } elsif ( $host ne "" ) { - $file = "$TopDir/pc/$host/LOG$ext"; + if ( !defined($In{num}) ) { + # get the latest LOG file + $file = ($bpc->sortedPCLogFiles($host))[0]; + } else { + $file = "$TopDir/pc/$host/LOG$ext"; + } $linkHosts = 1; } else { $file = "$LogDir/LOG$ext"; @@ -106,7 +108,8 @@ sub action } my($contentPre, $contentSub, $contentPost); $contentPre .= eval("qq{$Lang->{Log_File__file__comment}}"); - if ( defined($fh = BackupPC::FileZIO->open($file, 0, $compress)) ) { + if ( $file ne "" + && defined($fh = BackupPC::FileZIO->open($file, 0, $compress)) ) { $fh->utf8(1); my $mtimeStr = $bpc->timeStamp((stat($file))[9], 1); diff --git a/lib/BackupPC/Config/Meta.pm b/lib/BackupPC/Config/Meta.pm index c7258f1..cdd666d 100644 --- a/lib/BackupPC/Config/Meta.pm +++ b/lib/BackupPC/Config/Meta.pm @@ -139,7 +139,12 @@ use vars qw(%ConfigMeta); IncrKeepCnt => "integer", IncrKeepCntMin => "integer", IncrAgeMax => "float", + IncrLevels => { + type => "shortlist", + child => "integer", + }, PartialAgeMax => "float", + BackupsDisable => "integer", IncrFill => "boolean", RestoreInfoKeepCnt => "integer", ArchiveInfoKeepCnt => "integer", @@ -276,6 +281,7 @@ use vars qw(%ConfigMeta); RestorePostUserCmd => {type => "string", undefIfEmpty => 1}, ArchivePreUserCmd => {type => "string", undefIfEmpty => 1}, ArchivePostUserCmd => {type => "string", undefIfEmpty => 1}, + UserCmdCheckStatus => "integer", ClientNameAlias => {type => "string", undefIfEmpty => 1}, diff --git a/lib/BackupPC/Lang/de.pm b/lib/BackupPC/Lang/de.pm index f65da0b..7491166 100644 --- a/lib/BackupPC/Lang/de.pm +++ b/lib/BackupPC/Lang/de.pm @@ -640,6 +640,7 @@ Klicken Sie auf die Backupnummer um durch Dateien zu browsen und bei Bedarf wied Backup# Typ Filled + ENG Level Start Zeitpunkt Dauer/min Alter/Tage @@ -1312,6 +1313,22 @@ EOF $Lang{howLong_not_been_backed_up} = "Backup nicht erfolgreich"; $Lang{howLong_not_been_backed_up_for_days_days} = "kein Backup seit \$days Tagen"; +####################################################################### +# RSS strings (all ENGLISH currently) +####################################################################### +$Lang{RSS_Doc_Title} = "BackupPC Server"; +$Lang{RSS_Doc_Description} = "RSS feed for BackupPC"; +$Lang{RSS_Host_Summary} = < Backup# Type Filled + Level Start Date Duration/mins Age/days @@ -1303,6 +1304,22 @@ EOF $Lang{howLong_not_been_backed_up} = "not been backed up successfully"; $Lang{howLong_not_been_backed_up_for_days_days} = "not been backed up for \$days days"; +####################################################################### +# RSS strings +####################################################################### +$Lang{RSS_Doc_Title} = "BackupPC Server"; +$Lang{RSS_Doc_Description} = "RSS feed for BackupPC"; +$Lang{RSS_Host_Summary} = < Copia Nº Tipo Completo + ENG Level Fecha Inicio Duracion/mn Antigüedad/dias @@ -1308,6 +1309,22 @@ EOF $Lang{howLong_not_been_backed_up} = "no se le ha realizado una copia de seguridad con éxito"; $Lang{howLong_not_been_backed_up_for_days_days} = "no se le ha realizado una copia de seguridad durante \$days días"; +####################################################################### +# RSS strings +####################################################################### +$Lang{RSS_Doc_Title} = "BackupPC Server"; +$Lang{RSS_Doc_Description} = "RSS feed for BackupPC"; +$Lang{RSS_Host_Summary} = < Sauvegarde n° Type Fusionnée + ENG Level Date de démarrage Durée/min Âge/jours @@ -1304,6 +1305,22 @@ EOF $Lang{howLong_not_been_backed_up} = "jamais été sauvegardés"; $Lang{howLong_not_been_backed_up_for_days_days} = "pas été sauvegardés depuis \$days jours"; +####################################################################### +# RSS strings +####################################################################### +$Lang{RSS_Doc_Title} = "BackupPC Server"; +$Lang{RSS_Doc_Description} = "RSS feed for BackupPC"; +$Lang{RSS_Host_Summary} = <homepage di \$host. EOF @@ -302,7 +302,7 @@ $Lang{BackupPC__Start_Backup_Confirm_on__host} = "BackupPC: conferma avvio backu $Lang{Are_you_sure_start} = < -Si sta per avviare un bakcup \$type per \$host. +Si sta per avviare un backup \$type per \$host.
@@ -600,7 +600,7 @@ $Lang{Restore_Requested_on__hostDest} = "BackupPC: ripristino richiesto per \$ho $Lang{Reply_from_server_was___reply} = < -La risposta del server ` stata: \$reply +La risposta del server è stata: \$reply

Ritorna alla homepage di \$hostDest. EOF @@ -633,12 +633,13 @@ $Lang{Host__host_Backup_Summary2} = < \${h2("Prospetto backup")}

-Cliccare sul numero di bakcup per sfogliare e ripristinare i file di backup. +Cliccare sul numero di backup per sfogliare e ripristinare i file di backup.

+ @@ -889,7 +890,7 @@ $Lang{Restore___num_details_for__host2} = < - + + @@ -1317,6 +1318,22 @@ EOF $Lang{howLong_not_been_backed_up} = "(nog) niet succesvol gebackupt"; $Lang{howLong_not_been_backed_up_for_days_days} = "reeds sedert \$days dagen niet gebackupt"; +####################################################################### +# RSS strings +####################################################################### +$Lang{RSS_Doc_Title} = "BackupPC Server"; +$Lang{RSS_Doc_Description} = "RSS feed for BackupPC"; +$Lang{RSS_Host_Summary} = < + @@ -1307,6 +1308,22 @@ EOF $Lang{howLong_not_been_backed_up} = "não foi realizado nenhum backup com êxito"; $Lang{howLong_not_been_backed_up_for_days_days} = "não foi realizado nenhum backup durante \$days dias"; +####################################################################### +# RSS strings +####################################################################### +$Lang{RSS_Doc_Title} = "BackupPC Server"; +$Lang{RSS_Doc_Description} = "RSS feed for BackupPC"; +$Lang{RSS_Host_Summary} = <{Conf}{BackupPCUserVerify} && $> != (my $uid = (getpwnam($bpc->{Conf}{BackupPCUser}))[2]) ) { - print(STDERR "Wrong user: my userid is $>, instead of $uid" + print(STDERR "$0: Wrong user: my userid is $>, instead of $uid" . " ($bpc->{Conf}{BackupPCUser})\n"); + print(STDERR "Please su $bpc->{Conf}{BackupPCUser} first\n"); return; } return $bpc; @@ -1107,6 +1108,7 @@ sub cmdSystemOrEvalLong my($pid, $out, $allOut); local(*CHILD); + $? = 0; if ( (ref($cmd) eq "ARRAY" ? $cmd->[0] : $cmd) =~ /^\&/ ) { $cmd = join(" ", $cmd) if ( ref($cmd) eq "ARRAY" ); print(STDERR "cmdSystemOrEval: about to eval perl code $cmd\n") @@ -1218,4 +1220,62 @@ sub backupFileConfFix } } +# +# This is sort() compare function, used below. +# +# New client LOG names are LOG.MMYYYY. Old style names are +# LOG, LOG.0, LOG.1 etc. Sort them so new names are +# first, and newest to oldest. +# +sub compareLOGName +{ + my $na = $1 if ( $a =~ /LOG\.(\d+)(\.z)?$/ ); + my $nb = $1 if ( $b =~ /LOG\.(\d+)(\.z)?$/ ); + + $na = -1 if ( !defined($na) ); + $nb = -1 if ( !defined($nb) ); + + if ( length($na) >= 5 && length($nb) >= 5 ) { + # + # Both new style. Bigger numbers are more recent. + # + return $nb - $na; + } elsif ( length($na) >= 5 && length($nb) < 5 ) { + return -1; + } elsif ( length($na) < 5 && length($nb) >= 5 ) { + return 1; + } else { + # + # Both old style. Smaller numbers are more recent. + # + return $na - $nb; + } +} + +# +# Returns list of paths to a clients's (or main) LOG files, +# most recent first. +# +sub sortedPCLogFiles +{ + my($bpc, $host) = @_; + + my(@files, $dir); + + if ( $host ne "" ) { + $dir = "$bpc->{TopDir}/pc/$host"; + } else { + $dir = "$bpc->{LogDir}"; + } + if ( opendir(DIR, $dir) ) { + foreach my $file ( readdir(DIR) ) { + next if ( !-f "$dir/$file" ); + next if ( $file ne "LOG" && $file !~ /^LOG\.\d/ ); + push(@files, "$dir/$file"); + } + closedir(DIR); + } + return sort(compareLOGName @files); +} + 1; diff --git a/lib/BackupPC/PoolWrite.pm b/lib/BackupPC/PoolWrite.pm index 23eb788..54b2105 100644 --- a/lib/BackupPC/PoolWrite.pm +++ b/lib/BackupPC/PoolWrite.pm @@ -142,7 +142,19 @@ sub write my $fileName = $a->{fileCnt} < 0 ? $a->{base} : "$a->{base}_$a->{fileCnt}"; last if ( !-f $fileName ); + # + # Don't attempt to match pool files that already + # have too many hardlinks. Also, don't match pool + # files with only one link since starting in + # BackupPC v3.0, BackupPC_nightly could be running + # in parallel (and removing those files). This doesn't + # eliminate all possible race conditions, but just + # reduces the odds. Other design steps eliminate + # the remaining race conditions of linking vs + # removing. + # if ( (stat(_))[3] >= $a->{hardLinkMax} + || (stat(_))[3] <= 1 || !defined($fh = BackupPC::FileZIO->open($fileName, 0, $a->{compress})) ) { $a->{fileCnt}++; @@ -352,13 +364,6 @@ sub write } } - # - # Close the compare files - # - foreach my $f ( @{$a->{files}} ) { - $f->{fh}->close(); - } - if ( $a->{fileSize} == 0 ) { # # Simply create an empty file @@ -370,9 +375,21 @@ sub write } else { close(OUT); } + # + # Close the compare files + # + foreach my $f ( @{$a->{files}} ) { + $f->{fh}->close(); + } return (1, $a->{digest}, -s $a->{fileName}, $a->{errors}); } elsif ( defined($a->{fhOut}) ) { $a->{fhOut}->close(); + # + # Close the compare files + # + foreach my $f ( @{$a->{files}} ) { + $f->{fh}->close(); + } return (0, $a->{digest}, -s $a->{fileName}, $a->{errors}); } else { if ( @{$a->{files}} == 0 ) { @@ -390,12 +407,50 @@ sub write #} #push(@{$a->{errors}}, $str); } - #print(" Linking $a->{fileName} to $a->{files}[0]->{name}\n"); - if ( @{$a->{files}} && !link($a->{files}[0]->{name}, $a->{fileName}) ) { - push(@{$a->{errors}}, "Can't link $a->{fileName} to" - . " $a->{files}[0]->{name}\n"); + for ( my $i = 0 ; $i < @{$a->{files}} ; $i++ ) { + if ( link($a->{files}[$i]->{name}, $a->{fileName}) ) { + #print(" Linked $a->{fileName} to $a->{files}[$i]->{name}\n"); + # + # Close the compare files + # + foreach my $f ( @{$a->{files}} ) { + $f->{fh}->close(); + } + return (1, $a->{digest}, -s $a->{fileName}, $a->{errors}); + } + } + # + # We were unable to link to the pool. Either we're at the + # hardlink max, or the pool file got deleted. Recover by + # writing the matching file, since we still have an open + # handle. + # + for ( my $i = 0 ; $i < @{$a->{files}} ; $i++ ) { + if ( !$a->{files}[$i]->{fh}->rewind() ) { + push(@{$a->{errors}}, + "Unable to rewind $a->{files}[$i]->{name}" + . " for copy after link fail\n"); + next; + } + $a->{fhOut} = BackupPC::FileZIO->open($a->{fileName}, + 1, $a->{compress}); + if ( !defined($a->{fhOut}) ) { + push(@{$a->{errors}}, + "Unable to open $a->{fileName}" + . " for writing after link fail\n"); + } + $a->filePartialCopy($a->{files}[$i]->{fh}, $a->{fhOut}, + $a->{nWrite}); + $a->{fhOut}->close; + last; + } + # + # Close the compare files + # + foreach my $f ( @{$a->{files}} ) { + $f->{fh}->close(); } - return (1, $a->{digest}, -s $a->{fileName}, $a->{errors}); + return (0, $a->{digest}, -s $a->{fileName}, $a->{errors}); } } diff --git a/lib/BackupPC/Xfer/BackupPCd.pm b/lib/BackupPC/Xfer/BackupPCd.pm index e4e4b3c..a458429 100644 --- a/lib/BackupPC/Xfer/BackupPCd.pm +++ b/lib/BackupPC/Xfer/BackupPCd.pm @@ -117,8 +117,9 @@ sub start # # TODO: fix this message - just refer to the backup, not time? # - $incrDate = $bpc->timeStamp($t->{lastFull} - 3600, 1); - $logMsg = "incr backup started back to $incrDate for directory" + $incrDate = $bpc->timeStamp($t->{incrBaseTime} - 3600, 1); + $logMsg = "incr backup started back to $incrDate" + . " (backup #$t->{incrBaseBkupNum}) for directory" . " $t->{shareName}"; $incrFlag = 1; } diff --git a/lib/BackupPC/Xfer/Rsync.pm b/lib/BackupPC/Xfer/Rsync.pm index 8972a98..80d47a6 100644 --- a/lib/BackupPC/Xfer/Rsync.pm +++ b/lib/BackupPC/Xfer/Rsync.pm @@ -220,8 +220,9 @@ sub start $logMsg = "full backup started for directory $t->{shareName}"; } } else { - $incrDate = $bpc->timeStamp($t->{lastFull} - 3600, 1); - $logMsg = "incr backup started back to $incrDate for directory" + $incrDate = $bpc->timeStamp($t->{incrBaseTime}, 1); + $logMsg = "incr backup started back to $incrDate" + . " (backup #$t->{incrBaseBkupNum}) for directory" . " $t->{shareName}"; } @@ -245,7 +246,7 @@ sub start $fioArgs = { client => $t->{client}, share => $t->{shareName}, - viewNum => $t->{lastFullBkupNum}, + viewNum => $t->{incrBaseBkupNum}, partialNum => $t->{partialNum}, }; } diff --git a/lib/BackupPC/Xfer/Smb.pm b/lib/BackupPC/Xfer/Smb.pm index 03884a7..3839271 100644 --- a/lib/BackupPC/Xfer/Smb.pm +++ b/lib/BackupPC/Xfer/Smb.pm @@ -130,11 +130,13 @@ sub start } else { $timeStampFile = "$t->{outDir}/timeStamp.level0"; open(LEV0, ">", $timeStampFile) && close(LEV0); - utime($t->{lastFull} - 3600, $t->{lastFull} - 3600, $timeStampFile); + utime($t->{incrBaseTime} - 3600, $t->{incrBaseTime} - 3600, + $timeStampFile); $smbClientCmd = $conf->{SmbClientIncrCmd}; $logMsg = "incr backup started back to " - . $bpc->timeStamp($t->{lastFull} - 3600, 0) - . "for share $t->{shareName}"; + . $bpc->timeStamp($t->{incrBaseTime} - 3600, 0) + . " (backup #$t->{incrBaseBkupNum}) for share" + . " $t->{shareName}"; } } my $args = { @@ -261,6 +263,7 @@ sub readOutput || /^\s*Call timed out: server did not respond/i || /^\s*tree connect failed: ERRDOS - ERRnoaccess \(Access denied\.\)/ || /^\s*tree connect failed: NT_STATUS_BAD_NETWORK_NAME/ + || /^\s*NT_STATUS_INSUFF_SERVER_RESOURCES listing / ) { if ( $t->{hostError} eq "" ) { $t->{XferLOG}->write(\"This backup will fail because: $_\n"); diff --git a/lib/BackupPC/Xfer/Tar.pm b/lib/BackupPC/Xfer/Tar.pm index 84ebcb2..e9b2945 100644 --- a/lib/BackupPC/Xfer/Tar.pm +++ b/lib/BackupPC/Xfer/Tar.pm @@ -122,9 +122,10 @@ sub start $args = $conf->{TarFullArgs}; $logMsg = "full backup started for directory $t->{shareName}"; } else { - $incrDate = $bpc->timeStamp($t->{lastFull} - 3600, 1); + $incrDate = $bpc->timeStamp($t->{incrBaseTime} - 3600, 1); $args = $conf->{TarIncrArgs}; - $logMsg = "incr backup started back to $incrDate for directory" + $logMsg = "incr backup started back to $incrDate" + . " (backup #$t->{incrBaseBkupNum}) for directory" . " $t->{shareName}"; } push(@$tarClientCmd, split(/ +/, $args)); diff --git a/makeDist b/makeDist index af1092a..fc16eef 100755 --- a/makeDist +++ b/makeDist @@ -92,6 +92,7 @@ my @PerlSrc = qw( lib/BackupPC/CGI/RestoreFile.pm lib/BackupPC/CGI/RestoreInfo.pm lib/BackupPC/CGI/Restore.pm + lib/BackupPC/CGI/RSS.pm lib/BackupPC/CGI/StartServer.pm lib/BackupPC/CGI/StartStopBackup.pm lib/BackupPC/CGI/StopServer.pm -- 2.20.1
Numero backup Tipo Completo ENG Level Data avvio Durata (minuti) Età (giorni) Durata \$duration min
Numero file \$Restores[\$i]{nFiles}
Dimensione totale \${MB}MB
Tasso trasferimento \$MBperSecMB/s
Tasso trasferimento \${MBperSec}MB/s
Errori creazione tar \$Restores[\$i]{tarCreateErrs}
Errori trasferimento \$Restores[\$i]{xferErrs}
File log trasferimento @@ -1315,6 +1316,22 @@ EOF $Lang{howLong_not_been_backed_up} = "non e` riuscito"; $Lang{howLong_not_been_backed_up_for_days_days} = "risale a \$days giorni fa"; +####################################################################### +# RSS strings +####################################################################### +$Lang{RSS_Doc_Title} = "BackupPC Server"; +$Lang{RSS_Doc_Description} = "RSS feed for BackupPC"; +$Lang{RSS_Host_Summary} = < backup nr. Type Aangevuld ENG Level Startdatum Duurtijd in min. Lftd. in dagen Cópia Nº Tipo Completo ENG Level Data Início Duração/min Idade/dias