X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=bin%2FBackupPC_nightly;h=366345f43c6ec9b3719f879609caa0357cd4ab35;hp=b3a480263f70a86454cdb408aafca311cf71d789;hb=b81d2da5e16975674f011e4833337ac0fa24e0ea;hpb=5c6a6cc4f333ce44a9df62ab828b0b9341579f7c diff --git a/bin/BackupPC_nightly b/bin/BackupPC_nightly index b3a4802..366345f 100755 --- a/bin/BackupPC_nightly +++ b/bin/BackupPC_nightly @@ -20,6 +20,9 @@ # # -m Do monthly aging of per-PC log files and sending of email. # Otherise, BackupPC_nightly just does pool pruning. +# Since several BackupPC_nightly processes might run +# concurrently, just the first one is given the -m flag +# by BackupPC. # # The poolRangeStart and poolRangeEnd arguments are integers from 0 to 255. # These specify which parts of the pool to process. There are 256 2nd-level @@ -52,7 +55,7 @@ # #======================================================================== # -# Version 2.1.0beta1, released 9 Apr 2004. +# Version 3.0.0beta2, released 11 Nov 2006. # # See http://backuppc.sourceforge.net. # @@ -75,6 +78,14 @@ my $BinDir = $bpc->BinDir(); my %Conf = $bpc->Conf(); my(%Status, %Info, %Jobs, @BgQueue, @UserQueue, @CmdQueue); +# +# We delete unused pool files (link count 1) in sorted inode +# order by gathering batches. We delete the first half of +# each batch (ie: $PendingDeleteMax / 2 at a time). +# +my @PendingDelete; +my $PendingDeleteMax = 10240; + $bpc->ChildInit(); my %opts; @@ -104,13 +115,6 @@ if ( $opts{m} ) { eval($reply); } -########################################################################### -# When BackupPC_nightly starts, BackupPC will not run any simultaneous -# BackupPC_dump commands. We first do things that contend with -# BackupPC_dump, eg: aging per-PC log files etc. -########################################################################### -doPerPCLogFileAging() if ( $opts{m} ); - ########################################################################### # Get statistics on the pool, and remove files that have only one link. ########################################################################### @@ -127,6 +131,7 @@ my $fileCntRep; # total number of file names containing "_", ie: files my $fileRepMax; # worse case number of files that have repeated checksums # (ie: max(nnn+1) for all names xxxxxxxxxxxxxxxx_nnn) my $fileLinkMax; # maximum number of hardlinks on a pool file +my $fileLinkTotal; # total number of hardlinks on entire pool my $fileCntRename; # number of renamed files (to keep file numbering # contiguous) my %FixList; # list of paths that need to be renamed to avoid @@ -148,7 +153,8 @@ for my $pool ( qw(pool cpool) ) { $fileLinkMax = 0; $fileCntRename = 0; %FixList = (); - find({wanted => \&GetPoolStats}, "$TopDir/$pool/$dir"); + find({wanted => \&GetPoolStats}, "$TopDir/$pool/$dir") + if ( -d "$TopDir/$pool/$dir" ); my $kb = $blkCnt / 2; my $kbRm = $blkCntRm / 2; my $kb2 = $blkCnt2 / 2; @@ -193,65 +199,59 @@ for my $pool ( qw(pool cpool) ) { } print("BackupPC_stats $i = $pool,$fileCnt,$dirCnt,$kb,$kb2,$kbRm," . "$fileCntRm,$fileCntRep,$fileRepMax," - . "$fileCntRename,$fileLinkMax\n"); + . "$fileCntRename,$fileLinkMax,$fileLinkTotal\n"); } } +sleep(10); +processPendingDeletes(1); + ########################################################################### # Tell BackupPC that it is now ok to start running BackupPC_dump # commands. We are guaranteed that no BackupPC_link commands will # run since only a single CmdQueue command runs at a time, and -# that means we are safe. +# that means we are safe. As of 3.x this is irrelevant since +# BackupPC_dump runs independent of BackupPC_dump. ########################################################################### printf("BackupPC_nightly lock_off\n"); ########################################################################### -# Send email +# Send email and generation of backupInfo files for each backup ########################################################################### if ( $opts{m} ) { print("log BackupPC_nightly now running BackupPC_sendEmail\n"); - system("$BinDir/BackupPC_sendEmail") + system("$BinDir/BackupPC_sendEmail"); + doBackupInfoUpdate(); } # -# Do per-PC log file aging +# Update the backupInfo files based on the backups file. +# We do this just once a week (on Sun) since it is only +# needed for old backups with BackupPC <= 2.1.2. # -sub doPerPCLogFileAging +sub doBackupInfoUpdate { my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); - if ( $mday == 1 ) { - foreach my $host ( keys(%Status) ) { - my $lastLog = $Conf{MaxOldPerPCLogFiles} - 1; - unlink("$TopDir/pc/$host/LOG.$lastLog") - if ( -f "$TopDir/pc/$host/LOG.$lastLog" ); - unlink("$TopDir/pc/$host/LOG.$lastLog.z") - if ( -f "$TopDir/pc/$host/LOG.$lastLog.z" ); - for ( my $i = $lastLog - 1 ; $i >= 0 ; $i-- ) { - my $j = $i + 1; - if ( -f "$TopDir/pc/$host/LOG.$i" ) { - rename("$TopDir/pc/$host/LOG.$i", - "$TopDir/pc/$host/LOG.$j"); - } elsif ( -f "$TopDir/pc/$host/LOG.$i.z" ) { - rename("$TopDir/pc/$host/LOG.$i.z", - "$TopDir/pc/$host/LOG.$j.z"); - } - } + return if ( $wday != 0 ); + + foreach my $host ( sort(keys(%{$bpc->HostInfoRead()})) ) { + my @Backups = $bpc->BackupInfoRead($host); + + for ( my $i = 0 ; $i < @Backups ; $i++ ) { # - # Compress the log file LOG -> LOG.0.z (if enabled). - # Otherwise, just rename LOG -> LOG.0. + # BackupPC::Storage->backupInfoWrite won't overwrite + # an existing file # - BackupPC::FileZIO->compressCopy("$TopDir/pc/$host/LOG", - "$TopDir/pc/$host/LOG.0.z", - "$TopDir/pc/$host/LOG.0", - $Conf{CompressLevel}, 1); - open(LOG, ">", "$TopDir/pc/$host/LOG") && close(LOG); + BackupPC::Storage->backupInfoWrite("$TopDir/pc/$host", + $Backups[$i]{num}, + $Backups[$i]); } } } sub GetPoolStats { - my($nlinks, $nblocks) = (lstat($_))[3, 12]; + my($inode, $nlinks, $nblocks) = (lstat($_))[1, 3, 12]; if ( -d _ ) { $dirCnt++; @@ -262,7 +262,23 @@ sub GetPoolStats if ( $nlinks == 1 ) { $blkCntRm += $nblocks; $fileCntRm++; - unlink($_); + # + # Save the files for later batch deletion. + # + # This is so we can remove them in inode order, and additionally + # reduce any remaining chance of race condition of linking to + # pool files vs removing pool files. (Other aspects of the + # design should eliminate race conditions.) + # + my $fullPath = $File::Find::name; + push(@PendingDelete, { + inode => $inode, + path => $fullPath + } + ); + if ( @PendingDelete > $PendingDeleteMax ) { + processPendingDeletes(0); + } # # We must keep repeated files numbered sequential (ie: files # that have the same checksum are appended with _0, _1 etc). @@ -270,9 +286,8 @@ sub GetPoolStats # exists, or we remove any file of the form xxxx_nnn. We remember # the base name and fix it up later (not in the middle of find). # - my($baseName); - ($baseName = $File::Find::name) =~ s/_\d+$//; - $FixList{$baseName}++; + $fullPath =~ s/_\d+$//; + $FixList{$fullPath}++; } else { if ( /_(\d+)$/ ) { $fileRepMax = $1 + 1 if ( $fileRepMax <= $1 ); @@ -282,5 +297,26 @@ sub GetPoolStats $blkCnt += $nblocks; $blkCnt2 += $nblocks if ( $nlinks == 2 ); $fileLinkMax = $nlinks if ( $fileLinkMax < $nlinks ); + $fileLinkTotal += $nlinks - 1; + } +} + +sub processPendingDeletes +{ + my($doAll) = @_; + my @delete; + + if ( !$doAll ) { + @delete = splice(@PendingDelete, 0, $PendingDeleteMax / 2); + } else { + @delete = @PendingDelete; + @PendingDelete = (); + } + for my $f ( sort({ $a->{inode} <=> $b->{inode} } @delete) ) { + my($nlinks) = (lstat($f->{path}))[3]; + + next if ( $nlinks != 1 ); + # print("Deleting $f->{path} ($f->{inode})\n"); + unlink($f->{path}); } }