From 5c6a6cc4f333ce44a9df62ab828b0b9341579f7c Mon Sep 17 00:00:00 2001 From: cbarratt Date: Sat, 10 Apr 2004 06:52:44 +0000 Subject: [PATCH] Fixes for 2.1.0beta1: * The CSS definition has been removed from the config.pl file and is now a separate file, BackupPC_stnd.css. A new config variable, $Conf{CgiCSSFile}, gives the name of the CSS file to use. Suggested by Ender Mathias. * Fixed the filling of the host name select box for admins. The default $Conf{CgiNavBarAdminAllHosts} is now 1. Reported by Doug Lytle. * Cleaned up warning message for restore using rsync when checksum caching is on, but when file didn't have cached checksums. * Fixed BackupPC_archiveHost to support par2 (par2cmdline). Patch submitted by Jaco Bongers and adapted by Josh Marshall. * Improved stat() usage in BackupPC_nightly, plus some other cleanup, giving a significant performance improvement. Patch submitted by Wayne Scott. * Allow several BackupPC_nightly processes to run in parallel based on new $Conf{MaxBackupPCNightlyJobs} setting. This speeds up the traversal of the pool, reducing the overall run time for BackupPC_nightly. * Allow BackupPC_nightly to split the pool traversal across several nightly runs. This improves the running time per night, at the expense of a slight increase in disk storage as unused pool files might not be deleted for a couple of days. Controller by new config setting $Conf{BackupPCNightlyPeriod}. --- ChangeLog | 18 ++ bin/BackupPC | 240 +++++++++++++++++----- bin/BackupPC_archive | 6 +- bin/BackupPC_archiveHost | 17 +- bin/BackupPC_compressPool | 2 +- bin/BackupPC_dump | 2 +- bin/BackupPC_link | 2 +- bin/BackupPC_nightly | 295 +++++++++++++++++----------- bin/BackupPC_restore | 2 +- bin/BackupPC_sendEmail | 2 +- bin/BackupPC_serverMesg | 2 +- bin/BackupPC_tarCreate | 2 +- bin/BackupPC_tarExtract | 2 +- bin/BackupPC_trashClean | 2 +- bin/BackupPC_zcat | 2 +- bin/BackupPC_zipCreate | 2 +- cgi-bin/BackupPC_Admin | 2 +- conf/config.pl | 57 +++++- configure.pl | 2 +- doc-src/BackupPC.pod | 72 ++++++- lib/BackupPC/Attrib.pm | 2 +- lib/BackupPC/CGI/AdminOptions.pm | 2 +- lib/BackupPC/CGI/Archive.pm | 2 +- lib/BackupPC/CGI/ArchiveInfo.pm | 2 +- lib/BackupPC/CGI/Browse.pm | 2 +- lib/BackupPC/CGI/DirHistory.pm | 2 +- lib/BackupPC/CGI/EmailSummary.pm | 2 +- lib/BackupPC/CGI/GeneralInfo.pm | 2 +- lib/BackupPC/CGI/HostInfo.pm | 2 +- lib/BackupPC/CGI/LOGlist.pm | 2 +- lib/BackupPC/CGI/Lib.pm | 6 +- lib/BackupPC/CGI/Queue.pm | 6 +- lib/BackupPC/CGI/ReloadServer.pm | 2 +- lib/BackupPC/CGI/Restore.pm | 2 +- lib/BackupPC/CGI/RestoreFile.pm | 2 +- lib/BackupPC/CGI/RestoreInfo.pm | 2 +- lib/BackupPC/CGI/StartServer.pm | 2 +- lib/BackupPC/CGI/StartStopBackup.pm | 2 +- lib/BackupPC/CGI/StopServer.pm | 2 +- lib/BackupPC/CGI/Summary.pm | 2 +- lib/BackupPC/CGI/View.pm | 2 +- lib/BackupPC/FileZIO.pm | 2 +- lib/BackupPC/Lang/de.pm | 2 +- lib/BackupPC/Lang/en.pm | 2 +- lib/BackupPC/Lang/fr.pm | 2 +- lib/BackupPC/Lang/it.pm | 4 +- lib/BackupPC/Lib.pm | 14 +- lib/BackupPC/PoolWrite.pm | 2 +- lib/BackupPC/View.pm | 2 +- lib/BackupPC/Xfer/Archive.pm | 2 +- lib/BackupPC/Xfer/Rsync.pm | 2 +- lib/BackupPC/Xfer/RsyncDigest.pm | 2 +- lib/BackupPC/Xfer/RsyncFileIO.pm | 2 +- lib/BackupPC/Xfer/Smb.pm | 2 +- lib/BackupPC/Xfer/Tar.pm | 2 +- lib/BackupPC/Zip/FileMember.pm | 2 +- makeDist | 4 +- 57 files changed, 589 insertions(+), 240 deletions(-) diff --git a/ChangeLog b/ChangeLog index bcb84d8..d372dba 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,6 +33,24 @@ * Cleaned up warning message for restore using rsync when checksum caching is on, but when file didn't have cached checksums. +* Fixed BackupPC_archiveHost to support par2 (par2cmdline). + Patch submitted by Jaco Bongers and adapted by Josh Marshall. + +* Improved stat() usage in BackupPC_nightly, plus some other cleanup, + giving a significant performance improvement. Patch submitted by + Wayne Scott. + +* Allow several BackupPC_nightly processes to run in parallel based + on new $Conf{MaxBackupPCNightlyJobs} setting. This speeds up the + traversal of the pool, reducing the overall run time for + BackupPC_nightly. + +* Allow BackupPC_nightly to split the pool traversal across several + nightly runs. This improves the running time per night, at the expense + of a slight increase in disk storage as unused pool files might not + be deleted for a couple of days. Controller by new config setting + $Conf{BackupPCNightlyPeriod}. + #------------------------------------------------------------------------ # Version 2.1.0beta0, 20 Mar 2004 #------------------------------------------------------------------------ diff --git a/bin/BackupPC b/bin/BackupPC index a7247c8..585eec1 100755 --- a/bin/BackupPC +++ b/bin/BackupPC @@ -47,7 +47,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # @@ -173,6 +173,7 @@ my $SigName = ""; # Misc variables # my($RunNightlyWhenIdle, $FirstWakeup, $CmdJob, $ServerInetPort); +my($BackupPCNightlyJobs, $BackupPCNightlyLock); # # Complete the rest of the initialization @@ -374,14 +375,89 @@ sub Main_TryToRun_nightly $CmdQueueOn{$bpc->trashJob} = 1; } if ( keys(%Jobs) == $trashCleanRunning && $RunNightlyWhenIdle == 1 ) { - push(@CmdQueue, { - host => $bpc->adminJob, - user => "BackupPC", - reqTime => time, - cmd => ["$BinDir/BackupPC_nightly"], - }); - $CmdQueueOn{$bpc->adminJob} = 1; - $RunNightlyWhenIdle = 2; + + # + # Queue multiple nightly jobs based on the configuration + # + $Conf{MaxBackupPCNightlyJobs} = 1 + if ( $Conf{MaxBackupPCNightlyJobs} <= 0 ); + $Conf{BackupPCNightlyPeriod} = 1 + if ( $Conf{BackupPCNightlyPeriod} <= 0 ); + # + # Decide what subset of the 16 top-level directories 0..9a..f + # we run BackupPC_nightly on, based on $Conf{BackupPCNightlyPeriod}. + # If $Conf{BackupPCNightlyPeriod} == 1 then we run 0..15 every + # time. If $Conf{BackupPCNightlyPeriod} == 2 then we run + # 0..7 one night and 89a-f the next night. And so on. + # + # $Info{NightlyPhase} counts which night, from 0 to + # $Conf{BackupPCNightlyPeriod} - 1. + # + my $start = int($Info{NightlyPhase} * 16 + / $Conf{BackupPCNightlyPeriod}); + my $end = int(($Info{NightlyPhase} + 1) * 16 + / $Conf{BackupPCNightlyPeriod}); + $end = $start + 1 if ( $end <= $start ); + $Info{NightlyPhase}++; + $Info{NightlyPhase} = 0 if ( $end >= 16 ); + + # + # Zero out the data we expect to get from BackupPC_nightly. + # In the future if we want to split BackupPC_nightly over + # more than one night we will only zero out the portion + # that we are running right now. + # + for my $p ( qw(pool cpool) ) { + for ( my $i = $start ; $i < $end ; $i++ ) { + $Info{pool}{$p}[$i]{FileCnt} = 0; + $Info{pool}{$p}[$i]{DirCnt} = 0; + $Info{pool}{$p}[$i]{Kb} = 0; + $Info{pool}{$p}[$i]{Kb2} = 0; + $Info{pool}{$p}[$i]{KbRm} = 0; + $Info{pool}{$p}[$i]{FileCntRm} = 0; + $Info{pool}{$p}[$i]{FileCntRep} = 0; + $Info{pool}{$p}[$i]{FileRepMax} = 0; + $Info{pool}{$p}[$i]{FileCntRename} = 0; + $Info{pool}{$p}[$i]{FileLinkMax} = 0; + $Info{pool}{$p}[$i]{Time} = 0; + } + } + print(LOG $bpc->timeStamp, + sprintf("Running %d BackupPC_nightly jobs from %d..%d" + . " (out of 0..15)\n", + $Conf{MaxBackupPCNightlyJobs}, $start, $end - 1)); + + # + # Now queue the $Conf{MaxBackupPCNightlyJobs} jobs. + # The granularity on start and end is now 0..256. + # + $start *= 16; + $end *= 16; + my $start0 = $start; + for ( my $i = 0 ; $i < $Conf{MaxBackupPCNightlyJobs} ; $i++ ) { + # + # The first nightly job gets the -m option (does email, log aging). + # All jobs get the start and end options from 0..255 telling + # them which parts of the pool to traverse. + # + my $cmd = ["$BinDir/BackupPC_nightly"]; + push(@$cmd, "-m") if ( $i == 0 ); + push(@$cmd, $start); + $start = $start0 + int(($end - $start0) + * ($i + 1) / $Conf{MaxBackupPCNightlyJobs}); + push(@$cmd, $start - 1); + + my $job = $bpc->adminJob($i); + unshift(@CmdQueue, { + host => $job, + user => "BackupPC", + reqTime => time, + cmd => $cmd, + }); + $CmdQueueOn{$job} = 1; + } + $RunNightlyWhenIdle = 2; + } } @@ -397,7 +473,11 @@ sub Main_TryToRun_nightly sub Main_TryToRun_CmdQueue { my($req, $host); - if ( $CmdJob eq "" && @CmdQueue > 0 && $RunNightlyWhenIdle != 1 ) { + + while ( $CmdJob eq "" && @CmdQueue > 0 && $RunNightlyWhenIdle != 1 + || @CmdQueue > 0 && $RunNightlyWhenIdle == 2 + && $bpc->isAdminJob($CmdQueue[0]->{host}) + ) { local(*FH); $req = pop(@CmdQueue); @@ -435,8 +515,9 @@ sub Main_TryToRun_CmdQueue vec($FDread, $Jobs{$host}{fn}, 1) = 1; $Jobs{$host}{startTime} = time; $Jobs{$host}{reqTime} = $req->{reqTime}; - $cmd = join(" ", @$cmd); + $cmd = $bpc->execCmd2ShellCmd(@$cmd); $Jobs{$host}{cmd} = $cmd; + $Jobs{$host}{user} = $req->{user}; $Jobs{$host}{type} = $Status{$host}{type}; $Status{$host}{state} = "Status_link_running"; $Status{$host}{activeJob} = 1; @@ -444,6 +525,10 @@ sub Main_TryToRun_CmdQueue $CmdJob = $host if ( $host ne $bpc->trashJob ); $cmd =~ s/$BinDir\///g; print(LOG $bpc->timeStamp, "Running $cmd (pid=$pid)\n"); + if ( $cmd =~ /^BackupPC_nightly\s/ ) { + $BackupPCNightlyJobs++; + $BackupPCNightlyLock++; + } } } @@ -468,7 +553,7 @@ sub Main_TryToRun_Bg_or_User_Queue # # CmdJob and trashClean don't count towards MaxBackups / MaxUserBackups # - $nJobs-- if ( $CmdJob ne "" ); + $nJobs -= $BackupPCNightlyJobs if ( $CmdJob ne "" ); $nJobs-- if ( defined($Jobs{$bpc->trashJob} ) ); if ( $nJobs < $Conf{MaxBackups} + $Conf{MaxUserBackups} && @UserQueue > 0 ) { @@ -571,7 +656,7 @@ sub Main_TryToRun_Bg_or_User_Queue $Jobs{$host}{startTime} = time; $Jobs{$host}{reqTime} = $req->{reqTime}; $Jobs{$host}{userReq} = $req->{userReq}; - $Jobs{$host}{cmd} = join(" ", $progName, @args); + $Jobs{$host}{cmd} = $bpc->execCmd2ShellCmd($progName, @args); $Jobs{$host}{user} = $user; $Jobs{$host}{type} = $type; $Status{$host}{userReq} = $req->{userReq} @@ -922,33 +1007,32 @@ sub Main_Check_Job_Messages } } elsif ( $mesg =~ /^log\s+(.*)/ ) { print(LOG $bpc->timeStamp, "$1\n"); - } elsif ( $mesg =~ /^BackupPC_stats = (.*)/ ) { - my @f = split(/,/, $1); - $Info{"$f[0]FileCnt"} = $f[1]; - $Info{"$f[0]DirCnt"} = $f[2]; - $Info{"$f[0]Kb"} = $f[3]; - $Info{"$f[0]Kb2"} = $f[4]; - $Info{"$f[0]KbRm"} = $f[5]; - $Info{"$f[0]FileCntRm"} = $f[6]; - $Info{"$f[0]FileCntRep"} = $f[7]; - $Info{"$f[0]FileRepMax"} = $f[8]; - $Info{"$f[0]FileCntRename"} = $f[9]; - $Info{"$f[0]FileLinkMax"} = $f[10]; - $Info{"$f[0]Time"} = time; - printf(LOG "%s%s nightly clean removed %d files of" - . " size %.2fGB\n", - $bpc->timeStamp, ucfirst($f[0]), - $Info{"$f[0]FileCntRm"}, - $Info{"$f[0]KbRm"} / (1000 * 1024)); - printf(LOG "%s%s is %.2fGB, %d files (%d repeated, " - . "%d max chain, %d max links), %d directories\n", - $bpc->timeStamp, ucfirst($f[0]), - $Info{"$f[0]Kb"} / (1000 * 1024), - $Info{"$f[0]FileCnt"}, $Info{"$f[0]FileCntRep"}, - $Info{"$f[0]FileRepMax"}, - $Info{"$f[0]FileLinkMax"}, $Info{"$f[0]DirCnt"}); + } elsif ( $mesg =~ /^BackupPC_stats (\d+) = (.*)/ ) { + my $chunk = int($1 / 16); + my @f = split(/,/, $2); + $Info{pool}{$f[0]}[$chunk]{FileCnt} += $f[1]; + $Info{pool}{$f[0]}[$chunk]{DirCnt} += $f[2]; + $Info{pool}{$f[0]}[$chunk]{Kb} += $f[3]; + $Info{pool}{$f[0]}[$chunk]{Kb2} += $f[4]; + $Info{pool}{$f[0]}[$chunk]{KbRm} += $f[5]; + $Info{pool}{$f[0]}[$chunk]{FileCntRm} += $f[6]; + $Info{pool}{$f[0]}[$chunk]{FileCntRep} += $f[7]; + $Info{pool}{$f[0]}[$chunk]{FileRepMax} = $f[8] + if ( $Info{pool}{$f[0]}[$chunk]{FileRepMax} < $f[8] ); + $Info{pool}{$f[0]}[$chunk]{FileCntRename} += $f[9]; + $Info{pool}{$f[0]}[$chunk]{FileLinkMax} = $f[10] + if ( $Info{pool}{$f[0]}[$chunk]{FileLinkMax} < $f[10] ); + $Info{pool}{$f[0]}[$chunk]{Time} = time; } elsif ( $mesg =~ /^BackupPC_nightly lock_off/ ) { - $RunNightlyWhenIdle = 0; + $BackupPCNightlyLock--; + if ( $BackupPCNightlyLock == 0 ) { + # + # This means the last BackupPC_nightly is done with + # the pool clean, so it's to start running regular + # backups again. + # + $RunNightlyWhenIdle = 0; + } } elsif ( $mesg =~ /^processState\s+(.+)/ ) { $Jobs{$host}{processState} = $1; } elsif ( $mesg =~ /^link\s+(.+)/ ) { @@ -964,15 +1048,81 @@ sub Main_Check_Job_Messages if ( $nbytes <= 0 ) { close($Jobs{$host}{fh}); vec($FDread, $Jobs{$host}{fn}, 1) = 0; - if ( $CmdJob eq $host ) { + if ( $CmdJob eq $host || $bpc->isAdminJob($host) ) { my $cmd = $Jobs{$host}{cmd}; $cmd =~ s/$BinDir\///g; print(LOG $bpc->timeStamp, "Finished $host ($cmd)\n"); $Status{$host}{state} = "Status_idle"; $Status{$host}{endTime} = time; - $CmdJob = ""; - $RunNightlyWhenIdle = 0 if ( $cmd eq "BackupPC_nightly" - && $RunNightlyWhenIdle ); + if ( $cmd =~ /^BackupPC_nightly\s/ ) { + $BackupPCNightlyJobs--; + #print(LOG $bpc->timeStamp, "BackupPC_nightly done; now" + # . " have $BackupPCNightlyJobs running\n"); + if ( $BackupPCNightlyJobs <= 0 ) { + $BackupPCNightlyJobs = 0; + $RunNightlyWhenIdle = 0; + $CmdJob = ""; + # + # Combine the 16 per-directory results + # + for my $p ( qw(pool cpool) ) { + $Info{"${p}FileCnt"} = 0; + $Info{"${p}DirCnt"} = 0; + $Info{"${p}Kb"} = 0; + $Info{"${p}Kb2"} = 0; + $Info{"${p}KbRm"} = 0; + $Info{"${p}FileCntRm"} = 0; + $Info{"${p}FileCntRep"} = 0; + $Info{"${p}FileRepMax"} = 0; + $Info{"${p}FileCntRename"} = 0; + $Info{"${p}FileLinkMax"} = 0; + $Info{"${p}Time"} = 0; + for ( my $i = 0 ; $i < 16 ; $i++ ) { + $Info{"${p}FileCnt"} + += $Info{pool}{$p}[$i]{FileCnt}; + $Info{"${p}DirCnt"} + += $Info{pool}{$p}[$i]{DirCnt}; + $Info{"${p}Kb"} + += $Info{pool}{$p}[$i]{Kb}; + $Info{"${p}Kb2"} + += $Info{pool}{$p}[$i]{Kb2}; + $Info{"${p}KbRm"} + += $Info{pool}{$p}[$i]{KbRm}; + $Info{"${p}FileCntRm"} + += $Info{pool}{$p}[$i]{FileCntRm}; + $Info{"${p}FileCntRep"} + += $Info{pool}{$p}[$i]{FileCntRep}; + $Info{"${p}FileRepMax"} + = $Info{pool}{$p}[$i]{FileRepMax} + if ( $Info{"${p}FileRepMax"} < + $Info{pool}{$p}[$i]{FileRepMax} ); + $Info{"${p}FileCntRename"} + += $Info{pool}{$p}[$i]{FileCntRename}; + $Info{"${p}FileLinkMax"} + = $Info{pool}{$p}[$i]{FileLinkMax} + if ( $Info{"${p}FileLinkMax"} < + $Info{pool}{$p}[$i]{FileLinkMax} ); + $Info{"${p}Time"} = $Info{pool}{$p}[$i]{Time} + if ( $Info{"${p}Time"} < + $Info{pool}{$p}[$i]{Time} ); + } + printf(LOG "%s%s nightly clean removed %d files of" + . " size %.2fGB\n", + $bpc->timeStamp, ucfirst($p), + $Info{"${p}FileCntRm"}, + $Info{"${p}KbRm"} / (1000 * 1024)); + printf(LOG "%s%s is %.2fGB, %d files (%d repeated, " + . "%d max chain, %d max links), %d directories\n", + $bpc->timeStamp, ucfirst($p), + $Info{"${p}Kb"} / (1000 * 1024), + $Info{"${p}FileCnt"}, $Info{"${p}FileCntRep"}, + $Info{"${p}FileRepMax"}, + $Info{"${p}FileLinkMax"}, $Info{"${p}DirCnt"}); + } + } + } else { + $CmdJob = ""; + } } else { # # Queue BackupPC_link to complete the backup @@ -1118,6 +1268,8 @@ sub Main_Check_Client_Messages } } elsif ( $cmd =~ /^backup all$/ ) { QueueAllPCs(); + } elsif ( $cmd =~ /^BackupPC_nightly run$/ ) { + $RunNightlyWhenIdle = 1; } elsif ( $cmd =~ /^backup (\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ ) { my $hostIP = $1; $host = $2; @@ -1492,7 +1644,7 @@ sub HostsUpdate } foreach my $host ( sort(keys(%Status)) ) { next if ( $host eq $bpc->trashJob - || $host eq $bpc->adminJob + || $bpc->isAdminJob($host) || defined($Hosts->{$host}) || defined($Jobs{$host}) || $BgQueueOn{$host} diff --git a/bin/BackupPC_archive b/bin/BackupPC_archive index 386b355..3d9567f 100644 --- a/bin/BackupPC_archive +++ b/bin/BackupPC_archive @@ -8,10 +8,10 @@ # Usage: BackupPC_archive # # AUTHOR -# Craig Barratt +# Josh Marshall # # COPYRIGHT -# Copyright (C) 2001-2003 Craig Barratt +# Copyright (C) 2001-2004 Craig Barratt # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_archiveHost b/bin/BackupPC_archiveHost index 34a6e48..598347d 100755 --- a/bin/BackupPC_archiveHost +++ b/bin/BackupPC_archiveHost @@ -17,9 +17,10 @@ # # AUTHOR # Craig Barratt +# Josh Marshall # # COPYRIGHT -# Copyright (C) 2001-2003 Craig Barratt +# Copyright (C) 2001-2004 Craig Barratt # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,7 +38,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # @@ -125,10 +126,12 @@ if ( $ret ) { # ie: not a tape device). # if ( -d $outLoc && -x $parPath ) { - print("Running $parPath to create parity files\n"); - $ret = system("$parPath a -n $parfile $outLoc/$host.$bkupNum.tar$fileExt.par $outLoc/$host.$bkupNum.tar$fileExt.*"); - if ( $ret ) { - print("Error: $parPath failed\n"); - exit(1); + if ( $parfile != 0 ) { + print("Running $parPath to create parity files\n"); + $ret = system("$parPath c -r$parfile $outLoc/$host.$bkupNum.tar$fileExt.par2 $outLoc/$host.$bkupNum.tar$fileExt.*"); + if ( $ret ) { + print("Error: $parPath failed\n"); + exit(1); + } } } diff --git a/bin/BackupPC_compressPool b/bin/BackupPC_compressPool index cc7d5ba..b903268 100755 --- a/bin/BackupPC_compressPool +++ b/bin/BackupPC_compressPool @@ -49,7 +49,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_dump b/bin/BackupPC_dump index 2f3b7a3..3ac9ebc 100755 --- a/bin/BackupPC_dump +++ b/bin/BackupPC_dump @@ -70,7 +70,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_link b/bin/BackupPC_link index 290c9d9..8c75cde 100755 --- a/bin/BackupPC_link +++ b/bin/BackupPC_link @@ -39,7 +39,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_nightly b/bin/BackupPC_nightly index 5548738..b3a4802 100755 --- a/bin/BackupPC_nightly +++ b/bin/BackupPC_nightly @@ -4,20 +4,37 @@ # BackupPC_nightly: Nightly cleanup & statistics script. # # DESCRIPTION +# # BackupPC_nightly performs several administrative tasks: # -# - monthly aging of per-PC log files +# - monthly aging of per-PC log files (only with -m option) +# +# - pruning files from pool no longer used (ie: those with only one +# hard link). +# +# - sending email to users and administrators (only with -m option) # -# - pruning files from pool no longer used (ie: those with only one -# hard link). +# Usage: BackupPC_nightly [-m] poolRangeStart poolRangeEnd # -# - sending email to users and administrators. +# Flags: +# +# -m Do monthly aging of per-PC log files and sending of email. +# Otherise, BackupPC_nightly just does pool pruning. +# +# The poolRangeStart and poolRangeEnd arguments are integers from 0 to 255. +# These specify which parts of the pool to process. There are 256 2nd-level +# directories in the pool (0/0, 0/1, ..., f/e, f/f). BackupPC_nightly +# processes the given subset of this list (0 means 0/0, 255 means f/f). +# Therefore, arguments of 0 255 process the entire pool, 0 127 does +# the first half (ie: 0/0 through 7/f), 127 255 does the other half +# (eg: 8/0 through f/f) and 0 15 does just the first 1/16 of the pool +# (ie: 0/0 through 0/f). # # AUTHOR # Craig Barratt # # COPYRIGHT -# Copyright (C) 2001-2003 Craig Barratt +# Copyright (C) 2001-2004 Craig Barratt # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,7 +52,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # @@ -46,6 +63,7 @@ no utf8; use lib "/usr/local/BackupPC/lib"; use BackupPC::Lib; use BackupPC::FileZIO; +use Getopt::Std; use File::Find; use File::Path; @@ -55,56 +73,43 @@ die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); my $TopDir = $bpc->TopDir(); my $BinDir = $bpc->BinDir(); my %Conf = $bpc->Conf(); +my(%Status, %Info, %Jobs, @BgQueue, @UserQueue, @CmdQueue); $bpc->ChildInit(); -my $err = $bpc->ServerConnect($Conf{ServerHost}, $Conf{ServerPort}); -if ( $err ) { - print("Can't connect to server ($err)\n"); +my %opts; +if ( !getopts("m", \%opts) || @ARGV != 2 ) { + print("usage: $0 [-m] poolRangeStart poolRangeEnd\n"); exit(1); } -my $reply = $bpc->ServerMesg("status hosts"); -$reply = $1 if ( $reply =~ /(.*)/s ); -my(%Status, %Info, %Jobs, @BgQueue, @UserQueue, @CmdQueue); -eval($reply); +if ( $ARGV[0] !~ /^(\d+)$/ || $1 > 255 ) { + print("$0: bad poolRangeStart '$ARGV[0]'\n"); + exit(1); +} +my $poolRangeStart = $1; +if ( $ARGV[1] !~ /^(\d+)$/ || $1 > 255 ) { + print("$0: bad poolRangeEnd '$ARGV[1]'\n"); + exit(1); +} +my $poolRangeEnd = $1; + +if ( $opts{m} ) { + my $err = $bpc->ServerConnect($Conf{ServerHost}, $Conf{ServerPort}); + if ( $err ) { + print("Can't connect to server ($err)\n"); + exit(1); + } + my $reply = $bpc->ServerMesg("status hosts"); + $reply = $1 if ( $reply =~ /(.*)/s ); + eval($reply); +} ########################################################################### # When BackupPC_nightly starts, BackupPC will not run any simultaneous # BackupPC_dump commands. We first do things that contend with # BackupPC_dump, eg: aging per-PC log files etc. ########################################################################### - -# -# Do per-PC log file aging -# -my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); -if ( $mday == 1 ) { - foreach my $host ( keys(%Status) ) { - my $lastLog = $Conf{MaxOldPerPCLogFiles} - 1; - unlink("$TopDir/pc/$host/LOG.$lastLog") - if ( -f "$TopDir/pc/$host/LOG.$lastLog" ); - unlink("$TopDir/pc/$host/LOG.$lastLog.z") - if ( -f "$TopDir/pc/$host/LOG.$lastLog.z" ); - for ( my $i = $lastLog - 1 ; $i >= 0 ; $i-- ) { - my $j = $i + 1; - if ( -f "$TopDir/pc/$host/LOG.$i" ) { - rename("$TopDir/pc/$host/LOG.$i", "$TopDir/pc/$host/LOG.$j"); - } elsif ( -f "$TopDir/pc/$host/LOG.$i.z" ) { - rename("$TopDir/pc/$host/LOG.$i.z", - "$TopDir/pc/$host/LOG.$j.z"); - } - } - # - # Compress the log file LOG -> LOG.0.z (if enabled). - # Otherwise, just rename LOG -> LOG.0. - # - BackupPC::FileZIO->compressCopy("$TopDir/pc/$host/LOG", - "$TopDir/pc/$host/LOG.0.z", - "$TopDir/pc/$host/LOG.0", - $Conf{CompressLevel}, 1); - open(LOG, ">", "$TopDir/pc/$host/LOG") && close(LOG); - } -} +doPerPCLogFileAging() if ( $opts{m} ); ########################################################################### # Get statistics on the pool, and remove files that have only one link. @@ -126,53 +131,70 @@ my $fileCntRename; # number of renamed files (to keep file numbering # contiguous) my %FixList; # list of paths that need to be renamed to avoid # new holes +my @hexChars = qw(0 1 2 3 4 5 6 7 8 9 a b c d e f); + for my $pool ( qw(pool cpool) ) { - $fileCnt = 0; - $dirCnt = 0; - $blkCnt = 0; - $fileCntRm = 0; - $blkCntRm = 0; - $blkCnt2 = 0; - $fileCntRep = 0; - $fileRepMax = 0; - $fileLinkMax = 0; - $fileCntRename = 0; - %FixList = (); - find({wanted => \&GetPoolStats, no_chdir => 1}, "$TopDir/$pool"); - my $kb = $blkCnt / 2; - my $kbRm = $blkCntRm / 2; - my $kb2 = $blkCnt2 / 2; - - # - # Now make sure that files with repeated checksums are still - # sequentially numbered - # - foreach my $name ( sort(keys(%FixList)) ) { - my $rmCnt = $FixList{$name} + 1; - my $new = -1; - for ( my $old = -1 ; ; $old++ ) { - my $oldName = $name; - $oldName .= "_$old" if ( $old >= 0 ); - if ( !-f $oldName ) { - # - # We know we are done when we have missed at least - # the number of files that were removed from this - # base name, plus a couple just to be sure - # - last if ( $rmCnt-- <= 0 ); - next; + for ( my $i = $poolRangeStart ; $i <= $poolRangeEnd ; $i++ ) { + my $dir = "$hexChars[int($i / 16)]/$hexChars[$i % 16]"; + # print("Doing $pool/$dir\n") if ( ($i % 16) == 0 ); + $fileCnt = 0; + $dirCnt = 0; + $blkCnt = 0; + $fileCntRm = 0; + $blkCntRm = 0; + $blkCnt2 = 0; + $fileCntRep = 0; + $fileRepMax = 0; + $fileLinkMax = 0; + $fileCntRename = 0; + %FixList = (); + find({wanted => \&GetPoolStats}, "$TopDir/$pool/$dir"); + my $kb = $blkCnt / 2; + my $kbRm = $blkCntRm / 2; + my $kb2 = $blkCnt2 / 2; + + # + # Main BackupPC_nightly counts the top-level directory + # + $dirCnt++ if ( $opts{m} && -d "$TopDir/$pool" && $i == 0 ); + + # + # Also count the next level directories + # + $dirCnt++ if ( ($i % 16) == 0 + && -d "$TopDir/$pool/$hexChars[int($i / 16)]" ); + + # + # Now make sure that files with repeated checksums are still + # sequentially numbered + # + foreach my $name ( sort(keys(%FixList)) ) { + my $rmCnt = $FixList{$name} + 1; + my $new = -1; + for ( my $old = -1 ; ; $old++ ) { + my $oldName = $name; + $oldName .= "_$old" if ( $old >= 0 ); + if ( !-f $oldName ) { + # + # We know we are done when we have missed at least + # the number of files that were removed from this + # base name, plus a couple just to be sure + # + last if ( $rmCnt-- <= 0 ); + next; + } + my $newName = $name; + $newName .= "_$new" if ( $new >= 0 ); + $new++; + next if ( $oldName eq $newName ); + rename($oldName, $newName); + $fileCntRename++; } - my $newName = $name; - $newName .= "_$new" if ( $new >= 0 ); - $new++; - next if ( $oldName eq $newName ); - rename($oldName, $newName); - $fileCntRename++; } + print("BackupPC_stats $i = $pool,$fileCnt,$dirCnt,$kb,$kb2,$kbRm," + . "$fileCntRm,$fileCntRep,$fileRepMax," + . "$fileCntRename,$fileLinkMax\n"); } - print("BackupPC_stats = $pool,$fileCnt,$dirCnt,$kb,$kb2,$kbRm,$fileCntRm," - . "$fileCntRep,$fileRepMax,$fileCntRename," - . "$fileLinkMax\n"); } ########################################################################### @@ -186,42 +208,79 @@ printf("BackupPC_nightly lock_off\n"); ########################################################################### # Send email ########################################################################### -system("$BinDir/BackupPC_sendEmail"); +if ( $opts{m} ) { + print("log BackupPC_nightly now running BackupPC_sendEmail\n"); + system("$BinDir/BackupPC_sendEmail") +} + +# +# Do per-PC log file aging +# +sub doPerPCLogFileAging +{ + my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); + if ( $mday == 1 ) { + foreach my $host ( keys(%Status) ) { + my $lastLog = $Conf{MaxOldPerPCLogFiles} - 1; + unlink("$TopDir/pc/$host/LOG.$lastLog") + if ( -f "$TopDir/pc/$host/LOG.$lastLog" ); + unlink("$TopDir/pc/$host/LOG.$lastLog.z") + if ( -f "$TopDir/pc/$host/LOG.$lastLog.z" ); + for ( my $i = $lastLog - 1 ; $i >= 0 ; $i-- ) { + my $j = $i + 1; + if ( -f "$TopDir/pc/$host/LOG.$i" ) { + rename("$TopDir/pc/$host/LOG.$i", + "$TopDir/pc/$host/LOG.$j"); + } elsif ( -f "$TopDir/pc/$host/LOG.$i.z" ) { + rename("$TopDir/pc/$host/LOG.$i.z", + "$TopDir/pc/$host/LOG.$j.z"); + } + } + # + # Compress the log file LOG -> LOG.0.z (if enabled). + # Otherwise, just rename LOG -> LOG.0. + # + BackupPC::FileZIO->compressCopy("$TopDir/pc/$host/LOG", + "$TopDir/pc/$host/LOG.0.z", + "$TopDir/pc/$host/LOG.0", + $Conf{CompressLevel}, 1); + open(LOG, ">", "$TopDir/pc/$host/LOG") && close(LOG); + } + } +} sub GetPoolStats { - my($name) = $File::Find::name; - my($baseName) = ""; - my(@s); + my($nlinks, $nblocks) = (lstat($_))[3, 12]; - return if ( !-d && !-f ); - $dirCnt += -d; - $name = $1 if ( $name =~ /(.*)/ ); - @s = stat($name); - if ( $name =~ /(.*)_(\d+)$/ ) { - $baseName = $1; - if ( $s[3] != 1 ) { - $fileRepMax = $2 + 1 if ( $fileRepMax <= $2 ); - $fileCntRep++; - } + if ( -d _ ) { + $dirCnt++; + return; + } elsif ( ! -f _ ) { + return; } - if ( -f && $s[3] == 1 ) { - $blkCntRm += $s[12]; + if ( $nlinks == 1 ) { + $blkCntRm += $nblocks; $fileCntRm++; - unlink($name); - # - # We must keep repeated files numbered sequential (ie: files - # that have the same checksum are appended with _0, _1 etc). - # There are two cases: we remove the base file xxxx, but xxxx_0 - # exists, or we remove any file of the form xxxx_nnn. We remember - # the base name and fix it up later (not in the middle of find). - # - $baseName = $name if ( $baseName eq "" ); - $FixList{$baseName}++; + unlink($_); + # + # We must keep repeated files numbered sequential (ie: files + # that have the same checksum are appended with _0, _1 etc). + # There are two cases: we remove the base file xxxx, but xxxx_0 + # exists, or we remove any file of the form xxxx_nnn. We remember + # the base name and fix it up later (not in the middle of find). + # + my($baseName); + ($baseName = $File::Find::name) =~ s/_\d+$//; + $FixList{$baseName}++; } else { - $fileCnt += -f; - $blkCnt += $s[12]; - $blkCnt2 += $s[12] if ( -f && $s[3] == 2 ); - $fileLinkMax = $s[3] if ( $fileLinkMax < $s[3] ); + if ( /_(\d+)$/ ) { + $fileRepMax = $1 + 1 if ( $fileRepMax <= $1 ); + $fileCntRep++; + } + $fileCnt += 1; + $blkCnt += $nblocks; + $blkCnt2 += $nblocks if ( $nlinks == 2 ); + $fileLinkMax = $nlinks if ( $fileLinkMax < $nlinks ); } } diff --git a/bin/BackupPC_restore b/bin/BackupPC_restore index e0d1413..483615e 100755 --- a/bin/BackupPC_restore +++ b/bin/BackupPC_restore @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_sendEmail b/bin/BackupPC_sendEmail index 74e58a9..0f3b67b 100755 --- a/bin/BackupPC_sendEmail +++ b/bin/BackupPC_sendEmail @@ -31,7 +31,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_serverMesg b/bin/BackupPC_serverMesg index 3acb0d1..9ad000f 100755 --- a/bin/BackupPC_serverMesg +++ b/bin/BackupPC_serverMesg @@ -43,7 +43,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_tarCreate b/bin/BackupPC_tarCreate index 5407f58..64b2b49 100755 --- a/bin/BackupPC_tarCreate +++ b/bin/BackupPC_tarCreate @@ -51,7 +51,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_tarExtract b/bin/BackupPC_tarExtract index 5c10cac..fdef553 100755 --- a/bin/BackupPC_tarExtract +++ b/bin/BackupPC_tarExtract @@ -27,7 +27,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_trashClean b/bin/BackupPC_trashClean index 5b652f6..2d8f413 100755 --- a/bin/BackupPC_trashClean +++ b/bin/BackupPC_trashClean @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_zcat b/bin/BackupPC_zcat index e42e759..e758193 100755 --- a/bin/BackupPC_zcat +++ b/bin/BackupPC_zcat @@ -32,7 +32,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/bin/BackupPC_zipCreate b/bin/BackupPC_zipCreate index 362799e..74c4bf4 100755 --- a/bin/BackupPC_zipCreate +++ b/bin/BackupPC_zipCreate @@ -51,7 +51,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/cgi-bin/BackupPC_Admin b/cgi-bin/BackupPC_Admin index 0226c22..a3fbe87 100755 --- a/cgi-bin/BackupPC_Admin +++ b/cgi-bin/BackupPC_Admin @@ -39,7 +39,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/conf/config.pl b/conf/config.pl index 58991e1..d8bf145 100644 --- a/conf/config.pl +++ b/conf/config.pl @@ -140,6 +140,50 @@ $Conf{MaxUserBackups} = 4; # $Conf{MaxPendingCmds} = 10; +# +# How many BackupPC_nightly processes to run in parallel. +# +# Each night, at the first wakeup listed in $Conf{WakeupSchedule}, +# BackupPC_nightly is run. Its job is to remove unneeded files +# in the pool, ie: files that only have one link. To avoid race +# conditions, BackupPC_nightly runs only when there are no backups +# running, and no backups will start while it runs. +# +# So to reduce the elapsed time, you might want to increase this +# setting to run several BackupPC_nightly processes in parallel +# (eg: 4, or even 8). +# +$Conf{MaxBackupPCNightlyJobs} = 2; + +# +# How many days (runs) it takes BackupPC_nightly to traverse the +# entire pool. Normally this is 1, which means every night it runs, +# it does traverse the entire pool removing unused pool files. +# +# Other valid values are 2, 4, 8, 16. This causes BackupPC_nightly to +# traverse 1/2, 1/4, 1/8 or 1/16th of the pool each night, meaning it +# takes 2, 4, 8 or 16 days to completely traverse the pool. The +# advantage is that each night the running time of BackupPC_nightly +# is reduced roughly in proportion, since the total job is split +# over multiple days. The disadvantage is that unused pool files +# take longer to get deleted, which will slightly increase disk +# usage. +# +# Note that even when $Conf{BackupPCNightlyPeriod} > 1, BackupPC_nightly +# still runs every night. It just does less work each time it runs. +# +# Examples: +# +# $Conf{BackupPCNightlyPeriod} = 1; # entire pool is checked every night +# +# $Conf{BackupPCNightlyPeriod} = 2; # two days to complete pool check +# # (different half each night) +# +# $Conf{BackupPCNightlyPeriod} = 4; # four days to complete pool check +# # (different quarter each night) +# +$Conf{BackupPCNightlyPeriod} = 1; + # # Maximum number of log files we keep around in log directory. # These files are aged nightly. A setting of 14 means the log @@ -172,7 +216,7 @@ $Conf{DfCmd} = '$dfPath $topDir'; # $Conf{SplitPath} = '/usr/bin/split'; -$Conf{ParPath} = '/usr/bin/par'; +$Conf{ParPath} = '/usr/bin/par2'; $Conf{CatPath} = '/bin/cat'; $Conf{GzipPath} = '/bin/gzip'; $Conf{Bzip2Path} = '/usr/bin/bzip2'; @@ -1036,8 +1080,9 @@ $Conf{ArchiveComp} = 'gzip'; # # Archive Parity Files # -# The number of Parity Files to generate. -# Uses the commandline par available from +# The amount of Parity data to generate, as a percentage +# of the archive size. +# Uses the commandline par2 (par2cmdline) available from # http://parchive.sourceforge.net # # Only useful for file dumps. @@ -1064,14 +1109,14 @@ $Conf{ArchiveSplit} = 650; # $Installdir The installation directory of BackupPC # $tarCreatePath The path to BackupPC_tarCreate # $splitpath The path to the split program -# $parpath The path to the par program +# $parpath The path to the par2 program # $host The host to archive # $backupnumber The backup number of the host to archive # $compression The path to the compression program # $compext The extension assigned to the compression type # $splitsize The number of bytes to split archives into # $archiveloc The location to put the archive -# $parfile The number of par files to create +# $parfile The amount of parity data to create (percentage) # $Conf{ArchiveClientCmd} = '$Installdir/bin/BackupPC_archiveHost' . ' $tarCreatePath $splitpath $parpath $host $backupnumber' @@ -1312,7 +1357,7 @@ $Conf{MaxOldPerPCLogFiles} = 12; # $HostList list of hosts being archived # $BackupList list of backup numbers for the hosts being archived # $archiveloc location where the archive is sent to -# $parfile number of par files being generated +# $parfile amount of parity data being generated (percentage) # $compression compression program being used (eg: cat, gzip, bzip2) # $compext extension used for compression type (eg: raw, gz, bz2) # $splitsize size of the files that the archive creates diff --git a/configure.pl b/configure.pl index 1f787df..09d99b7 100755 --- a/configure.pl +++ b/configure.pl @@ -132,7 +132,7 @@ my %Programs = ( sendmail => "SendmailPath", hostname => "HostnamePath", split => "SplitPath", - 'parchive/par' => "ParPath", + par2 => "ParPath", cat => "CatPath", gzip => "GzipPath", bzip2 => "Bzip2Path", diff --git a/doc-src/BackupPC.pod b/doc-src/BackupPC.pod index 96e6c1f..75bcc63 100644 --- a/doc-src/BackupPC.pod +++ b/doc-src/BackupPC.pod @@ -267,6 +267,11 @@ releases of BackupPC: =item * +Develop a FAQ and move some significant parts of this document +to the FAQ (eg: ssh setup, this roadmap etc). Volunteers? + +=item * + Adding hardlink support to rsync. =item * @@ -299,14 +304,15 @@ file size, uid, gid), and better support for include and exclude. Currently smbclient incrementals only depend upon mtime, so deleted files or renamed files are not detected. FileSys::SmbClient would also allow resuming of incomplete full backups in the -same manner as rsync will. +same manner as rsync will. (I'm not sure if FileSys::SmbClient +has been updated for samba 3.x.) =item * -Support --listed-incremental or --incremental for tar, -so that incrementals will depend upon any attribute change (eg: exist, -mtime, file size, uid, gid), rather than just mtime. This will allow -tar to be to as capable as FileSys::SmbClient and rsync. +Possibly support --listed-incremental or --incremental for tar, so that +incrementals will depend upon any attribute change (eg: exist, mtime, +file size, uid, gid), rather than just mtime. This will allow tar to +be to as capable as FileSys::SmbClient and rsync. =item * @@ -319,6 +325,53 @@ the last, giving a continuous chain of differential dumps. =item * +Improve the warning messages about locked files on WinXX machines, +so that more file types than just outlook pst files will produce +warning emails (and configuration settings for specifying the +file extensions that produce email warngings). + +=item * + +Long term, support bare metal restore. For *nix machines there's +not a lot to do (althought rsync needs hardlink support). For +WinXX machines the file locking problem has to get resolved. +Plus ACL save/restore would need to be supported. This is +really long term. + +=item * + +Support client pull for restores. For example, BackupPC could +emulate an rsync server. That way you could boot a knoppix cd, +and you wouldn't need perl, BackupPC, ssh setup or anything on +the client to restore. You would just run an rsync command like: + + rsync -aH BackupPCServer::moduleName /path/to/emtpy/disk + +ModuleName could contain the client name, share name and backup number. +There would have to be some way of specifying the password; perhaps +the CGI could be used to "turn on" rsynd for a specific client and +a specific time period (eg: only listens for X minutes, only serves +up a specific client backup, accepts connections from a specific IP). +BackupPC listens for and serves the request; it's not a real rsyncd on +the server. + +=item * + +Add support for wget as an XferMethod. This would allow ftp and http +files to be backed up. This might be useful, for example, for backing +up the configuration of a router (via http), so that you have a +backup copy of all the router setup screens in case the router +fails. + +=item * + +Possibly support client push for backups, in addition to the +existing server pull. This would be helpful for clients behind +firewalls who can connect to the server, but the server cannot +see the client. + +=item * + More speculative: Storing binary file deltas (in fact, reverse deltas) for files that have the same name as a previous backup, but that aren't already in the pool. This will save storage for things like mailbox @@ -2256,6 +2309,15 @@ are using that file). Again, to avoid race conditions, BackupPC_nightly is only run when there are no BackupPC_dump or BackupPC_link processes running. +To improve the running time for BackupPC_nightly, several +BackupPC_nightly processes can be run concurrently, based on +the $Conf{MaxBackupPCNightlyJobs} setting. + +If BackupPC_nightly still takes too long to run, +$Conf{BackupPCNightlyPeriod} can be used to split +BackupPC_nightly's pool traversal across multiple nights, +proportionally reducing its runtime each night. + =back BackupPC also listens for TCP connections on $Conf{ServerPort}, which diff --git a/lib/BackupPC/Attrib.pm b/lib/BackupPC/Attrib.pm index d026d2d..73f6f1e 100644 --- a/lib/BackupPC/Attrib.pm +++ b/lib/BackupPC/Attrib.pm @@ -30,7 +30,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/AdminOptions.pm b/lib/BackupPC/CGI/AdminOptions.pm index dc378d3..ab9b752 100644 --- a/lib/BackupPC/CGI/AdminOptions.pm +++ b/lib/BackupPC/CGI/AdminOptions.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/Archive.pm b/lib/BackupPC/CGI/Archive.pm index 758cac3..590dd1f 100644 --- a/lib/BackupPC/CGI/Archive.pm +++ b/lib/BackupPC/CGI/Archive.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/ArchiveInfo.pm b/lib/BackupPC/CGI/ArchiveInfo.pm index 85198b1..4de942b 100644 --- a/lib/BackupPC/CGI/ArchiveInfo.pm +++ b/lib/BackupPC/CGI/ArchiveInfo.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/Browse.pm b/lib/BackupPC/CGI/Browse.pm index 6dffa35..ec30256 100644 --- a/lib/BackupPC/CGI/Browse.pm +++ b/lib/BackupPC/CGI/Browse.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/DirHistory.pm b/lib/BackupPC/CGI/DirHistory.pm index 39d8589..a7ea4bd 100644 --- a/lib/BackupPC/CGI/DirHistory.pm +++ b/lib/BackupPC/CGI/DirHistory.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/EmailSummary.pm b/lib/BackupPC/CGI/EmailSummary.pm index 6ae395a..f42e385 100644 --- a/lib/BackupPC/CGI/EmailSummary.pm +++ b/lib/BackupPC/CGI/EmailSummary.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/GeneralInfo.pm b/lib/BackupPC/CGI/GeneralInfo.pm index 88aeded..0323d5e 100644 --- a/lib/BackupPC/CGI/GeneralInfo.pm +++ b/lib/BackupPC/CGI/GeneralInfo.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/HostInfo.pm b/lib/BackupPC/CGI/HostInfo.pm index 3bdb1c1..7af47e9 100644 --- a/lib/BackupPC/CGI/HostInfo.pm +++ b/lib/BackupPC/CGI/HostInfo.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/LOGlist.pm b/lib/BackupPC/CGI/LOGlist.pm index 25e4060..05ae3da 100644 --- a/lib/BackupPC/CGI/LOGlist.pm +++ b/lib/BackupPC/CGI/LOGlist.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/Lib.pm b/lib/BackupPC/CGI/Lib.pm index 40950f8..1133b3a 100644 --- a/lib/BackupPC/CGI/Lib.pm +++ b/lib/BackupPC/CGI/Lib.pm @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # @@ -283,7 +283,9 @@ sub GetStatusInfo eval($reply); # ignore status related to admin and trashClean jobs if ( $status =~ /\bhosts\b/ ) { - delete($Status{$bpc->adminJob}); + foreach my $host ( grep(/admin/, keys(%Status)) ) { + delete($Status{$host}) if ( $bpc->isAdminJob($host) ); + } delete($Status{$bpc->trashJob}); } } diff --git a/lib/BackupPC/CGI/Queue.pm b/lib/BackupPC/CGI/Queue.pm index ba30e19..6b0a9cc 100644 --- a/lib/BackupPC/CGI/Queue.pm +++ b/lib/BackupPC/CGI/Queue.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # @@ -71,12 +71,12 @@ EOF while ( @CmdQueue ) { my $req = pop(@CmdQueue); my $reqTime = timeStamp2($req->{reqTime}); - (my $cmd = $req->{cmd}[0]) =~ s/$BinDir\///; + (my $cmd = $bpc->execCmd2ShellCmd(@{$req->{cmd}})) =~ s/$BinDir\///; $strCmd .= < ${HostLink($req->{host})} $reqTime $req->{user} - $cmd $req->{cmd}[0] + $cmd EOF } my $content = eval ( "qq{$Lang->{Backup_Queue_Summary}}"); diff --git a/lib/BackupPC/CGI/ReloadServer.pm b/lib/BackupPC/CGI/ReloadServer.pm index edbf490..27a9add 100644 --- a/lib/BackupPC/CGI/ReloadServer.pm +++ b/lib/BackupPC/CGI/ReloadServer.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/Restore.pm b/lib/BackupPC/CGI/Restore.pm index f00ce59..11ed25b 100644 --- a/lib/BackupPC/CGI/Restore.pm +++ b/lib/BackupPC/CGI/Restore.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/RestoreFile.pm b/lib/BackupPC/CGI/RestoreFile.pm index e543797..52f3e90 100644 --- a/lib/BackupPC/CGI/RestoreFile.pm +++ b/lib/BackupPC/CGI/RestoreFile.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/RestoreInfo.pm b/lib/BackupPC/CGI/RestoreInfo.pm index 8b7fbc9..a693795 100644 --- a/lib/BackupPC/CGI/RestoreInfo.pm +++ b/lib/BackupPC/CGI/RestoreInfo.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/StartServer.pm b/lib/BackupPC/CGI/StartServer.pm index c12fd89..f40910b 100644 --- a/lib/BackupPC/CGI/StartServer.pm +++ b/lib/BackupPC/CGI/StartServer.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/StartStopBackup.pm b/lib/BackupPC/CGI/StartStopBackup.pm index 8e6df0e..dc345a6 100644 --- a/lib/BackupPC/CGI/StartStopBackup.pm +++ b/lib/BackupPC/CGI/StartStopBackup.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/StopServer.pm b/lib/BackupPC/CGI/StopServer.pm index b05d3b1..dd22921 100644 --- a/lib/BackupPC/CGI/StopServer.pm +++ b/lib/BackupPC/CGI/StopServer.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/Summary.pm b/lib/BackupPC/CGI/Summary.pm index fd2444c..6c7459e 100644 --- a/lib/BackupPC/CGI/Summary.pm +++ b/lib/BackupPC/CGI/Summary.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/CGI/View.pm b/lib/BackupPC/CGI/View.pm index c24b80d..fc99fee 100644 --- a/lib/BackupPC/CGI/View.pm +++ b/lib/BackupPC/CGI/View.pm @@ -28,7 +28,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/FileZIO.pm b/lib/BackupPC/FileZIO.pm index a465d06..84fca36 100644 --- a/lib/BackupPC/FileZIO.pm +++ b/lib/BackupPC/FileZIO.pm @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/Lang/de.pm b/lib/BackupPC/Lang/de.pm index 2391c46..1e487fc 100644 --- a/lib/BackupPC/Lang/de.pm +++ b/lib/BackupPC/Lang/de.pm @@ -259,7 +259,7 @@ EOF $Lang{BackupPC_Archive2_parity} = < - Anzahl Parität-Dateien + Prozentsatz Paritätsdaten (0 = keine, 5 = Standard) EOF diff --git a/lib/BackupPC/Lang/en.pm b/lib/BackupPC/Lang/en.pm index da86c75..eb629f1 100644 --- a/lib/BackupPC/Lang/en.pm +++ b/lib/BackupPC/Lang/en.pm @@ -254,7 +254,7 @@ EOF $Lang{BackupPC_Archive2_parity} = < - Number of Parity Files + Percentage of Parity Data (0 = disable, 5 = typical) EOF diff --git a/lib/BackupPC/Lang/fr.pm b/lib/BackupPC/Lang/fr.pm index 032f6d3..04ddc6b 100644 --- a/lib/BackupPC/Lang/fr.pm +++ b/lib/BackupPC/Lang/fr.pm @@ -254,7 +254,7 @@ EOF $Lang{BackupPC_Archive2_parity} = < - Nombre de fichiers de parité + Pourcentage des données de parité (0 = désactivé, 5 = typique) EOF diff --git a/lib/BackupPC/Lang/it.pm b/lib/BackupPC/Lang/it.pm index 65bcd36..b821a80 100644 --- a/lib/BackupPC/Lang/it.pm +++ b/lib/BackupPC/Lang/it.pm @@ -1,6 +1,6 @@ #!/bin/perl # -# $Id: it.pm,v 1.5 2004/03/29 19:05:24 cbarratt Exp $ +# $Id: it.pm,v 1.6 2004/04/10 06:52:47 cbarratt Exp $ # # Italian i18n file # @@ -262,7 +262,7 @@ EOF $Lang{BackupPC_Archive2_parity} = < - Numero di file di parità + ENG Percentage of Parity Data (0 = disable, 5 = typical) EOF diff --git a/lib/BackupPC/Lib.pm b/lib/BackupPC/Lib.pm index 49354b6..ad6af10 100644 --- a/lib/BackupPC/Lib.pm +++ b/lib/BackupPC/Lib.pm @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # @@ -59,7 +59,7 @@ sub new TopDir => $topDir || '/data/BackupPC', BinDir => $installDir || '/usr/local/BackupPC', LibDir => $installDir || '/usr/local/BackupPC', - Version => '2.1.0beta0', + Version => '2.1.0beta1', BackupFields => [qw( num type startTime endTime nFiles size nFilesExist sizeExist nFilesNew sizeNew @@ -132,7 +132,15 @@ sub Lang sub adminJob { - return " admin "; + my($bpc, $num) = @_; + return " admin " if ( !$num ); + return " admin$num "; +} + +sub isAdminJob +{ + my($bpc, $str) = @_; + return $str =~ /^ admin/; } sub trashJob diff --git a/lib/BackupPC/PoolWrite.pm b/lib/BackupPC/PoolWrite.pm index a992e7a..b8d0cdf 100644 --- a/lib/BackupPC/PoolWrite.pm +++ b/lib/BackupPC/PoolWrite.pm @@ -56,7 +56,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/View.pm b/lib/BackupPC/View.pm index 5006f87..3083dfc 100644 --- a/lib/BackupPC/View.pm +++ b/lib/BackupPC/View.pm @@ -31,7 +31,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/Xfer/Archive.pm b/lib/BackupPC/Xfer/Archive.pm index dc3427d..ba86fb3 100644 --- a/lib/BackupPC/Xfer/Archive.pm +++ b/lib/BackupPC/Xfer/Archive.pm @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/Xfer/Rsync.pm b/lib/BackupPC/Xfer/Rsync.pm index 5708731..338ca56 100644 --- a/lib/BackupPC/Xfer/Rsync.pm +++ b/lib/BackupPC/Xfer/Rsync.pm @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/Xfer/RsyncDigest.pm b/lib/BackupPC/Xfer/RsyncDigest.pm index 1d07147..be0b24b 100644 --- a/lib/BackupPC/Xfer/RsyncDigest.pm +++ b/lib/BackupPC/Xfer/RsyncDigest.pm @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/Xfer/RsyncFileIO.pm b/lib/BackupPC/Xfer/RsyncFileIO.pm index 2c25f62..b028b65 100644 --- a/lib/BackupPC/Xfer/RsyncFileIO.pm +++ b/lib/BackupPC/Xfer/RsyncFileIO.pm @@ -12,7 +12,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/Xfer/Smb.pm b/lib/BackupPC/Xfer/Smb.pm index b362953..ebdff99 100644 --- a/lib/BackupPC/Xfer/Smb.pm +++ b/lib/BackupPC/Xfer/Smb.pm @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/Xfer/Tar.pm b/lib/BackupPC/Xfer/Tar.pm index 1de40ff..b72e5e6 100644 --- a/lib/BackupPC/Xfer/Tar.pm +++ b/lib/BackupPC/Xfer/Tar.pm @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/lib/BackupPC/Zip/FileMember.pm b/lib/BackupPC/Zip/FileMember.pm index bdc4b45..debd5c3 100644 --- a/lib/BackupPC/Zip/FileMember.pm +++ b/lib/BackupPC/Zip/FileMember.pm @@ -33,7 +33,7 @@ # #======================================================================== # -# Version 2.1.0beta0, released 20 Mar 2004. +# Version 2.1.0beta1, released 9 Apr 2004. # # See http://backuppc.sourceforge.net. # diff --git a/makeDist b/makeDist index 3e8ff3a..ebddca8 100755 --- a/makeDist +++ b/makeDist @@ -42,8 +42,8 @@ use Getopt::Std; umask(0022); -my $Version = "2.1.0beta0_CVS"; -my $ReleaseDate = "29 Mar 2004"; +my $Version = "2.1.0beta1"; +my $ReleaseDate = "9 Apr 2004"; my $DistDir = "dist/BackupPC-$Version"; my @PerlSrc = qw( -- 2.20.1