-#!/bin/perl
+#!/usr/bin/perl
#============================================================= -*-perl-*-
#
# BackupPC_nightly: Nightly cleanup & statistics script.
#
# -m Do monthly aging of per-PC log files and sending of email.
# Otherise, BackupPC_nightly just does pool pruning.
+# Since several BackupPC_nightly processes might run
+# concurrently, just the first one is given the -m flag
+# by BackupPC.
#
# The poolRangeStart and poolRangeEnd arguments are integers from 0 to 255.
# These specify which parts of the pool to process. There are 256 2nd-level
# Craig Barratt <cbarratt@users.sourceforge.net>
#
# COPYRIGHT
-# Copyright (C) 2001-2004 Craig Barratt
+# Copyright (C) 2001-2009 Craig Barratt
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
#
#========================================================================
#
-# Version 2.1.0beta2, released 23 May 2004.
+# Version 3.2.0, released 31 Jul 2010.
#
# See http://backuppc.sourceforge.net.
#
use strict;
no utf8;
use lib "/usr/local/BackupPC/lib";
-use BackupPC::Lib;
+use BackupPC::Lib qw( :BPC_DT_ALL );
use BackupPC::FileZIO;
use Getopt::Std;
-use File::Find;
use File::Path;
use Data::Dumper;
my %Conf = $bpc->Conf();
my(%Status, %Info, %Jobs, @BgQueue, @UserQueue, @CmdQueue);
+#
+# We delete unused pool files (link count 1) in sorted inode
+# order by gathering batches. We delete the first half of
+# each batch (ie: $PendingDeleteMax / 2 at a time).
+#
+my @PendingDelete;
+my $PendingDeleteMax = 10240;
+
$bpc->ChildInit();
my %opts;
eval($reply);
}
-###########################################################################
-# When BackupPC_nightly starts, BackupPC will not run any simultaneous
-# BackupPC_dump commands. We first do things that contend with
-# BackupPC_dump, eg: aging per-PC log files etc.
-###########################################################################
-doPerPCLogFileAging() if ( $opts{m} );
-
###########################################################################
# Get statistics on the pool, and remove files that have only one link.
###########################################################################
my $fileRepMax; # worse case number of files that have repeated checksums
# (ie: max(nnn+1) for all names xxxxxxxxxxxxxxxx_nnn)
my $fileLinkMax; # maximum number of hardlinks on a pool file
+my $fileLinkTotal; # total number of hardlinks on entire pool
my $fileCntRename; # number of renamed files (to keep file numbering
# contiguous)
my %FixList; # list of paths that need to be renamed to avoid
$fileLinkMax = 0;
$fileCntRename = 0;
%FixList = ();
- find({wanted => \&GetPoolStats}, "$TopDir/$pool/$dir");
+ $bpc->find({wanted => \&GetPoolStats}, "$TopDir/$pool/$dir")
+ if ( -d "$TopDir/$pool/$dir" );
my $kb = $blkCnt / 2;
my $kbRm = $blkCntRm / 2;
my $kb2 = $blkCnt2 / 2;
$dirCnt++ if ( ($i % 16) == 0
&& -d "$TopDir/$pool/$hexChars[int($i / 16)]" );
+ #
+ # We need to process all pending deletes before we do the
+ # renames
+ #
+ if ( @PendingDelete ) {
+ sleep(1);
+ processPendingDeletes(1);
+ }
+
#
# Now make sure that files with repeated checksums are still
# sequentially numbered
}
print("BackupPC_stats $i = $pool,$fileCnt,$dirCnt,$kb,$kb2,$kbRm,"
. "$fileCntRm,$fileCntRep,$fileRepMax,"
- . "$fileCntRename,$fileLinkMax\n");
+ . "$fileCntRename,$fileLinkMax,$fileLinkTotal\n");
}
}
+sleep(1);
+processPendingDeletes(1);
+
###########################################################################
# Tell BackupPC that it is now ok to start running BackupPC_dump
# commands. We are guaranteed that no BackupPC_link commands will
# run since only a single CmdQueue command runs at a time, and
-# that means we are safe.
+# that means we are safe. As of 3.x this is irrelevant since
+# BackupPC_dump runs independent of BackupPC_dump.
###########################################################################
printf("BackupPC_nightly lock_off\n");
###########################################################################
-# Send email
+# Send email and generation of backupInfo files for each backup
###########################################################################
if ( $opts{m} ) {
print("log BackupPC_nightly now running BackupPC_sendEmail\n");
- system("$BinDir/BackupPC_sendEmail")
+ system("$BinDir/BackupPC_sendEmail");
+ doBackupInfoUpdate();
}
#
-# Do per-PC log file aging
+# Update the backupInfo files based on the backups file.
+# We do this just once a week (on Sun) since it is only
+# needed for old backups with BackupPC <= 2.1.2.
#
-sub doPerPCLogFileAging
+sub doBackupInfoUpdate
{
my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
- if ( $mday == 1 ) {
- foreach my $host ( keys(%Status) ) {
- my $lastLog = $Conf{MaxOldPerPCLogFiles} - 1;
- unlink("$TopDir/pc/$host/LOG.$lastLog")
- if ( -f "$TopDir/pc/$host/LOG.$lastLog" );
- unlink("$TopDir/pc/$host/LOG.$lastLog.z")
- if ( -f "$TopDir/pc/$host/LOG.$lastLog.z" );
- for ( my $i = $lastLog - 1 ; $i >= 0 ; $i-- ) {
- my $j = $i + 1;
- if ( -f "$TopDir/pc/$host/LOG.$i" ) {
- rename("$TopDir/pc/$host/LOG.$i",
- "$TopDir/pc/$host/LOG.$j");
- } elsif ( -f "$TopDir/pc/$host/LOG.$i.z" ) {
- rename("$TopDir/pc/$host/LOG.$i.z",
- "$TopDir/pc/$host/LOG.$j.z");
- }
- }
+ return if ( $wday != 0 );
+
+ foreach my $host ( sort(keys(%{$bpc->HostInfoRead()})) ) {
+ my @Backups = $bpc->BackupInfoRead($host);
+
+ for ( my $i = 0 ; $i < @Backups ; $i++ ) {
#
- # Compress the log file LOG -> LOG.0.z (if enabled).
- # Otherwise, just rename LOG -> LOG.0.
+ # BackupPC::Storage->backupInfoWrite won't overwrite
+ # an existing file
#
- BackupPC::FileZIO->compressCopy("$TopDir/pc/$host/LOG",
- "$TopDir/pc/$host/LOG.0.z",
- "$TopDir/pc/$host/LOG.0",
- $Conf{CompressLevel}, 1);
- open(LOG, ">", "$TopDir/pc/$host/LOG") && close(LOG);
+ BackupPC::Storage->backupInfoWrite("$TopDir/pc/$host",
+ $Backups[$i]{num},
+ $Backups[$i]);
}
}
}
sub GetPoolStats
{
- my($nlinks, $nblocks) = (lstat($_))[3, 12];
+ my($file, $fullPath) = @_;
+ my($inode, $nlinks, $nblocks) = (lstat($file))[1, 3, 12];
if ( -d _ ) {
$dirCnt++;
if ( $nlinks == 1 ) {
$blkCntRm += $nblocks;
$fileCntRm++;
- unlink($_);
+ #
+ # Save the files for later batch deletion.
+ #
+ # This is so we can remove them in inode order, and additionally
+ # reduce any remaining chance of race condition of linking to
+ # pool files vs removing pool files. (Other aspects of the
+ # design should eliminate race conditions.)
+ #
+ push(@PendingDelete, {
+ inode => $inode,
+ path => $fullPath
+ }
+ );
+ if ( @PendingDelete > $PendingDeleteMax ) {
+ processPendingDeletes(0);
+ }
#
# We must keep repeated files numbered sequential (ie: files
# that have the same checksum are appended with _0, _1 etc).
# exists, or we remove any file of the form xxxx_nnn. We remember
# the base name and fix it up later (not in the middle of find).
#
- my($baseName);
- ($baseName = $File::Find::name) =~ s/_\d+$//;
- $FixList{$baseName}++;
+ $fullPath =~ s/_\d+$//;
+ $FixList{$fullPath}++;
} else {
- if ( /_(\d+)$/ ) {
+ if ( $file =~ /_(\d+)$/ ) {
$fileRepMax = $1 + 1 if ( $fileRepMax <= $1 );
$fileCntRep++;
}
$blkCnt += $nblocks;
$blkCnt2 += $nblocks if ( $nlinks == 2 );
$fileLinkMax = $nlinks if ( $fileLinkMax < $nlinks );
+ $fileLinkTotal += $nlinks - 1;
+ }
+}
+
+sub processPendingDeletes
+{
+ my($doAll) = @_;
+ my @delete;
+
+ if ( !$doAll ) {
+ @delete = splice(@PendingDelete, 0, $PendingDeleteMax / 2);
+ } else {
+ @delete = @PendingDelete;
+ @PendingDelete = ();
+ }
+ for my $f ( sort({ $a->{inode} <=> $b->{inode} } @delete) ) {
+ my($nlinks) = (lstat($f->{path}))[3];
+
+ next if ( $nlinks != 1 );
+ # print("Deleting $f->{path} ($f->{inode})\n");
+ unlink($f->{path});
}
}
projects / BackupPC.git / blobdiff