-#!/bin/perl
+#!/usr/bin/perl
#============================================================= -*-perl-*-
#
# BackupPC_nightly: Nightly cleanup & statistics script.
# Craig Barratt <cbarratt@users.sourceforge.net>
#
# COPYRIGHT
-# Copyright (C) 2001-2004 Craig Barratt
+# Copyright (C) 2001-2009 Craig Barratt
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
#
#========================================================================
#
-# Version 2.1.0, released 20 Jun 2004.
+# Version 3.2.0, released 31 Jul 2010.
#
# See http://backuppc.sourceforge.net.
#
use strict;
no utf8;
use lib "/usr/local/BackupPC/lib";
-use BackupPC::Lib;
+use BackupPC::Lib qw( :BPC_DT_ALL );
use BackupPC::FileZIO;
use Getopt::Std;
-use File::Find;
use File::Path;
use Data::Dumper;
my %Conf = $bpc->Conf();
my(%Status, %Info, %Jobs, @BgQueue, @UserQueue, @CmdQueue);
+#
+# We delete unused pool files (link count 1) in sorted inode
+# order by gathering batches. We delete the first half of
+# each batch (ie: $PendingDeleteMax / 2 at a time).
+#
+my @PendingDelete;
+my $PendingDeleteMax = 10240;
+
$bpc->ChildInit();
my %opts;
eval($reply);
}
-###########################################################################
-# When BackupPC_nightly starts, BackupPC will not run any simultaneous
-# BackupPC_dump commands. We first do things that contend with
-# BackupPC_dump, eg: aging per-PC log files etc.
-###########################################################################
-doPerPCLogFileAging() if ( $opts{m} );
-
###########################################################################
# Get statistics on the pool, and remove files that have only one link.
###########################################################################
$fileLinkMax = 0;
$fileCntRename = 0;
%FixList = ();
- find({wanted => \&GetPoolStats}, "$TopDir/$pool/$dir")
+ $bpc->find({wanted => \&GetPoolStats}, "$TopDir/$pool/$dir")
if ( -d "$TopDir/$pool/$dir" );
my $kb = $blkCnt / 2;
my $kbRm = $blkCntRm / 2;
$dirCnt++ if ( ($i % 16) == 0
&& -d "$TopDir/$pool/$hexChars[int($i / 16)]" );
+ #
+ # We need to process all pending deletes before we do the
+ # renames
+ #
+ if ( @PendingDelete ) {
+ sleep(1);
+ processPendingDeletes(1);
+ }
+
#
# Now make sure that files with repeated checksums are still
# sequentially numbered
}
}
+sleep(1);
+processPendingDeletes(1);
+
###########################################################################
# Tell BackupPC that it is now ok to start running BackupPC_dump
# commands. We are guaranteed that no BackupPC_link commands will
# run since only a single CmdQueue command runs at a time, and
-# that means we are safe.
+# that means we are safe. As of 3.x this is irrelevant since
+# BackupPC_dump runs independent of BackupPC_dump.
###########################################################################
printf("BackupPC_nightly lock_off\n");
doBackupInfoUpdate();
}
-#
-# Do per-PC log file aging
-#
-sub doPerPCLogFileAging
-{
- my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
- if ( $mday == 1 ) {
- foreach my $host ( keys(%Status) ) {
- my $lastLog = $Conf{MaxOldPerPCLogFiles} - 1;
- unlink("$TopDir/pc/$host/LOG.$lastLog")
- if ( -f "$TopDir/pc/$host/LOG.$lastLog" );
- unlink("$TopDir/pc/$host/LOG.$lastLog.z")
- if ( -f "$TopDir/pc/$host/LOG.$lastLog.z" );
- for ( my $i = $lastLog - 1 ; $i >= 0 ; $i-- ) {
- my $j = $i + 1;
- if ( -f "$TopDir/pc/$host/LOG.$i" ) {
- rename("$TopDir/pc/$host/LOG.$i",
- "$TopDir/pc/$host/LOG.$j");
- } elsif ( -f "$TopDir/pc/$host/LOG.$i.z" ) {
- rename("$TopDir/pc/$host/LOG.$i.z",
- "$TopDir/pc/$host/LOG.$j.z");
- }
- }
- #
- # Compress the log file LOG -> LOG.0.z (if enabled).
- # Otherwise, just rename LOG -> LOG.0.
- #
- BackupPC::FileZIO->compressCopy("$TopDir/pc/$host/LOG",
- "$TopDir/pc/$host/LOG.0.z",
- "$TopDir/pc/$host/LOG.0",
- $Conf{CompressLevel}, 1);
- open(LOG, ">", "$TopDir/pc/$host/LOG") && close(LOG);
- }
- }
-}
-
#
# Update the backupInfo files based on the backups file.
# We do this just once a week (on Sun) since it is only
sub GetPoolStats
{
- my($nlinks, $nblocks) = (lstat($_))[3, 12];
+ my($file, $fullPath) = @_;
+ my($inode, $nlinks, $nblocks) = (lstat($file))[1, 3, 12];
if ( -d _ ) {
$dirCnt++;
if ( $nlinks == 1 ) {
$blkCntRm += $nblocks;
$fileCntRm++;
- unlink($_);
+ #
+ # Save the files for later batch deletion.
+ #
+ # This is so we can remove them in inode order, and additionally
+ # reduce any remaining chance of race condition of linking to
+ # pool files vs removing pool files. (Other aspects of the
+ # design should eliminate race conditions.)
+ #
+ push(@PendingDelete, {
+ inode => $inode,
+ path => $fullPath
+ }
+ );
+ if ( @PendingDelete > $PendingDeleteMax ) {
+ processPendingDeletes(0);
+ }
#
# We must keep repeated files numbered sequential (ie: files
# that have the same checksum are appended with _0, _1 etc).
# exists, or we remove any file of the form xxxx_nnn. We remember
# the base name and fix it up later (not in the middle of find).
#
- my($baseName);
- ($baseName = $File::Find::name) =~ s/_\d+$//;
- $FixList{$baseName}++;
+ $fullPath =~ s/_\d+$//;
+ $FixList{$fullPath}++;
} else {
- if ( /_(\d+)$/ ) {
+ if ( $file =~ /_(\d+)$/ ) {
$fileRepMax = $1 + 1 if ( $fileRepMax <= $1 );
$fileCntRep++;
}
$fileLinkTotal += $nlinks - 1;
}
}
+
+sub processPendingDeletes
+{
+ my($doAll) = @_;
+ my @delete;
+
+ if ( !$doAll ) {
+ @delete = splice(@PendingDelete, 0, $PendingDeleteMax / 2);
+ } else {
+ @delete = @PendingDelete;
+ @PendingDelete = ();
+ }
+ for my $f ( sort({ $a->{inode} <=> $b->{inode} } @delete) ) {
+ my($nlinks) = (lstat($f->{path}))[3];
+
+ next if ( $nlinks != 1 );
+ # print("Deleting $f->{path} ($f->{inode})\n");
+ unlink($f->{path});
+ }
+}