2 #============================================================= -*-perl-*-
4 # BackupPC_nightly: Nightly cleanup & statistics script.
8 # BackupPC_nightly performs several administrative tasks:
10 # - monthly aging of per-PC log files (only with -m option)
12 # - pruning files from pool no longer used (ie: those with only one
15 # - sending email to users and administrators (only with -m option)
17 # Usage: BackupPC_nightly [-m] poolRangeStart poolRangeEnd
21 # -m Do monthly aging of per-PC log files and sending of email.
22 # Otherise, BackupPC_nightly just does pool pruning.
23 # Since several BackupPC_nightly processes might run
24 # concurrently, just the first one is given the -m flag
27 # The poolRangeStart and poolRangeEnd arguments are integers from 0 to 255.
28 # These specify which parts of the pool to process. There are 256 2nd-level
29 # directories in the pool (0/0, 0/1, ..., f/e, f/f). BackupPC_nightly
30 # processes the given subset of this list (0 means 0/0, 255 means f/f).
31 # Therefore, arguments of 0 255 process the entire pool, 0 127 does
32 # the first half (ie: 0/0 through 7/f), 127 255 does the other half
33 # (eg: 8/0 through f/f) and 0 15 does just the first 1/16 of the pool
34 # (ie: 0/0 through 0/f).
37 # Craig Barratt <cbarratt@users.sourceforge.net>
40 # Copyright (C) 2001-2004 Craig Barratt
42 # This program is free software; you can redistribute it and/or modify
43 # it under the terms of the GNU General Public License as published by
44 # the Free Software Foundation; either version 2 of the License, or
45 # (at your option) any later version.
47 # This program is distributed in the hope that it will be useful,
48 # but WITHOUT ANY WARRANTY; without even the implied warranty of
49 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
50 # GNU General Public License for more details.
52 # You should have received a copy of the GNU General Public License
53 # along with this program; if not, write to the Free Software
54 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
56 #========================================================================
58 # Version 3.0.0alpha, released 23 Jan 2006.
60 # See http://backuppc.sourceforge.net.
62 #========================================================================
66 use lib "/usr/local/BackupPC/lib";
68 use BackupPC::FileZIO;
75 die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) );
76 my $TopDir = $bpc->TopDir();
77 my $BinDir = $bpc->BinDir();
78 my %Conf = $bpc->Conf();
79 my(%Status, %Info, %Jobs, @BgQueue, @UserQueue, @CmdQueue);
82 # We delete unused pool files (link count 1) in sorted inode
83 # order by gathering batches. We delete the first half of
84 # each batch (ie: $PendingDeleteMax / 2 at a time).
87 my $PendingDeleteMax = 10240;
92 if ( !getopts("m", \%opts) || @ARGV != 2 ) {
93 print("usage: $0 [-m] poolRangeStart poolRangeEnd\n");
96 if ( $ARGV[0] !~ /^(\d+)$/ || $1 > 255 ) {
97 print("$0: bad poolRangeStart '$ARGV[0]'\n");
100 my $poolRangeStart = $1;
101 if ( $ARGV[1] !~ /^(\d+)$/ || $1 > 255 ) {
102 print("$0: bad poolRangeEnd '$ARGV[1]'\n");
105 my $poolRangeEnd = $1;
108 my $err = $bpc->ServerConnect($Conf{ServerHost}, $Conf{ServerPort});
110 print("Can't connect to server ($err)\n");
113 my $reply = $bpc->ServerMesg("status hosts");
114 $reply = $1 if ( $reply =~ /(.*)/s );
118 ###########################################################################
119 # Get statistics on the pool, and remove files that have only one link.
120 ###########################################################################
122 my $fileCnt; # total number of files
123 my $dirCnt; # total number of directories
124 my $blkCnt; # total block size of files
125 my $fileCntRm; # total number of removed files
126 my $blkCntRm; # total block size of removed files
127 my $blkCnt2; # total block size of files with just 2 links
128 # (ie: files that only occur once among all backups)
129 my $fileCntRep; # total number of file names containing "_", ie: files
130 # that have repeated md5 checksums
131 my $fileRepMax; # worse case number of files that have repeated checksums
132 # (ie: max(nnn+1) for all names xxxxxxxxxxxxxxxx_nnn)
133 my $fileLinkMax; # maximum number of hardlinks on a pool file
134 my $fileLinkTotal; # total number of hardlinks on entire pool
135 my $fileCntRename; # number of renamed files (to keep file numbering
137 my %FixList; # list of paths that need to be renamed to avoid
139 my @hexChars = qw(0 1 2 3 4 5 6 7 8 9 a b c d e f);
141 for my $pool ( qw(pool cpool) ) {
142 for ( my $i = $poolRangeStart ; $i <= $poolRangeEnd ; $i++ ) {
143 my $dir = "$hexChars[int($i / 16)]/$hexChars[$i % 16]";
144 # print("Doing $pool/$dir\n") if ( ($i % 16) == 0 );
156 find({wanted => \&GetPoolStats}, "$TopDir/$pool/$dir")
157 if ( -d "$TopDir/$pool/$dir" );
158 my $kb = $blkCnt / 2;
159 my $kbRm = $blkCntRm / 2;
160 my $kb2 = $blkCnt2 / 2;
163 # Main BackupPC_nightly counts the top-level directory
165 $dirCnt++ if ( $opts{m} && -d "$TopDir/$pool" && $i == 0 );
168 # Also count the next level directories
170 $dirCnt++ if ( ($i % 16) == 0
171 && -d "$TopDir/$pool/$hexChars[int($i / 16)]" );
174 # Now make sure that files with repeated checksums are still
175 # sequentially numbered
177 foreach my $name ( sort(keys(%FixList)) ) {
178 my $rmCnt = $FixList{$name} + 1;
180 for ( my $old = -1 ; ; $old++ ) {
182 $oldName .= "_$old" if ( $old >= 0 );
183 if ( !-f $oldName ) {
185 # We know we are done when we have missed at least
186 # the number of files that were removed from this
187 # base name, plus a couple just to be sure
189 last if ( $rmCnt-- <= 0 );
193 $newName .= "_$new" if ( $new >= 0 );
195 next if ( $oldName eq $newName );
196 rename($oldName, $newName);
200 print("BackupPC_stats $i = $pool,$fileCnt,$dirCnt,$kb,$kb2,$kbRm,"
201 . "$fileCntRm,$fileCntRep,$fileRepMax,"
202 . "$fileCntRename,$fileLinkMax,$fileLinkTotal\n");
207 processPendingDeletes(1);
209 ###########################################################################
210 # Tell BackupPC that it is now ok to start running BackupPC_dump
211 # commands. We are guaranteed that no BackupPC_link commands will
212 # run since only a single CmdQueue command runs at a time, and
213 # that means we are safe. As of 3.x this is irrelevant since
214 # BackupPC_dump runs independent of BackupPC_dump.
215 ###########################################################################
216 printf("BackupPC_nightly lock_off\n");
218 ###########################################################################
219 # Send email and generation of backupInfo files for each backup
220 ###########################################################################
222 print("log BackupPC_nightly now running BackupPC_sendEmail\n");
223 system("$BinDir/BackupPC_sendEmail");
224 doBackupInfoUpdate();
228 # Update the backupInfo files based on the backups file.
229 # We do this just once a week (on Sun) since it is only
230 # needed for old backups with BackupPC <= 2.1.2.
232 sub doBackupInfoUpdate
234 my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
235 return if ( $wday != 0 );
237 foreach my $host ( sort(keys(%{$bpc->HostInfoRead()})) ) {
238 my @Backups = $bpc->BackupInfoRead($host);
240 for ( my $i = 0 ; $i < @Backups ; $i++ ) {
242 # BackupPC::Storage->backupInfoWrite won't overwrite
245 BackupPC::Storage->backupInfoWrite("$TopDir/pc/$host",
254 my($inode, $nlinks, $nblocks) = (lstat($_))[1, 3, 12];
262 if ( $nlinks == 1 ) {
263 $blkCntRm += $nblocks;
266 # Save the files for later batch deletion.
268 # This is so we can remove them in inode order, and additionally
269 # reduce any remaining chance of race condition of linking to
270 # pool files vs removing pool files. (Other aspects of the
271 # design should eliminate race conditions.)
273 my $fullPath = $File::Find::name;
274 push(@PendingDelete, {
279 if ( @PendingDelete > $PendingDeleteMax ) {
280 processPendingDeletes(0);
283 # We must keep repeated files numbered sequential (ie: files
284 # that have the same checksum are appended with _0, _1 etc).
285 # There are two cases: we remove the base file xxxx, but xxxx_0
286 # exists, or we remove any file of the form xxxx_nnn. We remember
287 # the base name and fix it up later (not in the middle of find).
289 $fullPath =~ s/_\d+$//;
290 $FixList{$fullPath}++;
293 $fileRepMax = $1 + 1 if ( $fileRepMax <= $1 );
298 $blkCnt2 += $nblocks if ( $nlinks == 2 );
299 $fileLinkMax = $nlinks if ( $fileLinkMax < $nlinks );
300 $fileLinkTotal += $nlinks - 1;
304 sub processPendingDeletes
310 @delete = splice(@PendingDelete, 0, $PendingDeleteMax / 2);
312 @delete = @PendingDelete;
315 for my $f ( sort({ $a->{inode} <=> $b->{inode} } @delete) ) {
316 my($nlinks) = (lstat($f->{path}))[3];
318 next if ( $nlinks != 1 );
319 # print("Deleting $f->{path} ($f->{inode})\n");