1 #============================================================= -*-perl-*-
3 # BackupPC::Xfer::Rsync package
7 # This library defines a BackupPC::Xfer::Rsync class for managing
8 # the rsync-based transport of backup data from the client.
11 # Craig Barratt <cbarratt@users.sourceforge.net>
14 # Copyright (C) 2002-2003 Craig Barratt
16 # This program is free software; you can redistribute it and/or modify
17 # it under the terms of the GNU General Public License as published by
18 # the Free Software Foundation; either version 2 of the License, or
19 # (at your option) any later version.
21 # This program is distributed in the hope that it will be useful,
22 # but WITHOUT ANY WARRANTY; without even the implied warranty of
23 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24 # GNU General Public License for more details.
26 # You should have received a copy of the GNU General Public License
27 # along with this program; if not, write to the Free Software
28 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
30 #========================================================================
32 # Version 2.1.0, released 20 Jun 2004.
34 # See http://backuppc.sourceforge.net.
36 #========================================================================
38 package BackupPC::Xfer::Rsync;
42 use BackupPC::Xfer::RsyncFileIO;
44 use vars qw( $RsyncLibOK $RsyncLibErr );
47 eval "use File::RsyncP;";
50 # Rsync module doesn't exist.
53 $RsyncLibErr = "File::RsyncP module doesn't exist";
56 # Note: also update configure.pl when this version number is changed!
58 if ( $File::RsyncP::VERSION < 0.51 ) {
60 $RsyncLibErr = "File::RsyncP module version too old: need 0.50";
69 my($class, $bpc, $args) = @_;
71 return if ( !$RsyncLibOK );
75 conf => { $bpc->Conf },
104 foreach my $arg ( keys(%$args) ) {
105 $t->{$arg} = $args->{$arg};
118 my $conf = $t->{conf};
119 my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
120 $incrDate, $argList, $fioArgs);
123 # We add a slash to the share name we pass to rsync
125 ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{/};
127 if ( $t->{type} eq "restore" ) {
128 $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
129 $rsyncArgs = $conf->{RsyncRestoreArgs};
130 my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
131 $remoteDir =~ s{//+}{/}g;
132 $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
134 client => $t->{bkupSrcHost},
135 share => $t->{bkupSrcShare},
136 viewNum => $t->{bkupSrcNum},
137 fileList => $t->{fileList},
139 $logMsg = "restore started below directory $t->{shareName}"
140 . " to host $t->{host}";
143 # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
144 # into a hash of arrays of files, and $conf->{RsyncShareName}
147 $bpc->backupFileConfFix($conf, "RsyncShareName");
149 if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
150 my(@inc, @exc, %incDone, %excDone);
151 foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
153 # If the user wants to just include /home/craig, then
154 # we need to do create include/exclude pairs at
156 # --include /home --exclude /*
157 # --include /home/craig --exclude /home/*
159 # It's more complex if the user wants to include multiple
160 # deep paths. For example, if they want /home/craig and
161 # /var/log, then we need this mouthfull:
162 # --include /home --include /var --exclude /*
163 # --include /home/craig --exclude /home/*
164 # --include /var/log --exclude /var/*
166 # To make this easier we do all the includes first and all
167 # of the excludes at the end (hopefully they commute).
172 if ( $file eq "/" ) {
174 # This is a special case: if the user specifies
175 # "/" then just include it and don't exclude "/*".
177 push(@inc, $file) if ( !$incDone{$file} );
181 while ( $file =~ m{^/([^/]*)(.*)} ) {
184 if ( $file eq "/" ) {
186 # preserve a tailing slash
191 push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
192 $excDone{"$f/*"} = 1;
194 push(@inc, $f) if ( !$incDone{$f} );
198 foreach my $file ( @inc ) {
199 push(@fileList, "--include=$file");
201 foreach my $file ( @exc ) {
202 push(@fileList, "--exclude=$file");
205 if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
206 foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
209 # just append additional exclude lists onto the end
211 push(@fileList, "--exclude=$file");
214 if ( $t->{type} eq "full" ) {
215 if ( $t->{partialNum} ) {
216 $logMsg = "full backup started for directory $t->{shareName};"
217 . " updating partial $t->{partialNum}";
219 $logMsg = "full backup started for directory $t->{shareName}";
222 $incrDate = $bpc->timeStamp($t->{lastFull} - 3600, 1);
223 $logMsg = "incr backup started back to $incrDate for directory"
224 . " $t->{shareName}";
228 # A full dump is implemented with --ignore-times: this causes all
229 # files to be checksummed, even if the attributes are the same.
230 # That way all the file contents are checked, but you get all
231 # the efficiencies of rsync: only files deltas need to be
232 # transferred, even though it is a full dump.
234 $rsyncArgs = $conf->{RsyncArgs};
235 $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList );
236 $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
237 if ( $t->{type} eq "full" );
238 $rsyncClientCmd = $conf->{RsyncClientCmd};
239 $argList = ['--server', '--sender', @$rsyncArgs,
240 '.', $t->{shareNameSlash}];
242 client => $t->{client},
243 share => $t->{shareName},
244 viewNum => $t->{lastFullBkupNum},
245 partialNum => $t->{partialNum},
250 # Merge variables into $rsyncClientCmd
254 hostIP => $t->{hostIP},
255 client => $t->{client},
256 shareName => $t->{shareName},
257 shareNameSlash => $t->{shareNameSlash},
258 rsyncPath => $conf->{RsyncClientPath},
259 sshPath => $conf->{SshPath},
262 $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args);
265 # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
266 # module, which handles all the special BackupPC file storage
267 # (compression, mangling, hardlinks, special files, attributes etc).
269 $t->{rsyncClientCmd} = $rsyncClientCmd;
270 $t->{rs} = File::RsyncP->new({
271 logLevel => $t->{logLevel} || $conf->{RsyncLogLevel},
274 $bpc->cmdExecOrEval($rsyncClientCmd, $args);
276 rsyncCmdType => "full",
277 rsyncArgs => $rsyncArgs,
278 timeout => $conf->{ClientTimeout},
279 doPartial => defined($t->{partialNum}) ? 1 : undef,
284 $t->{XferLOG}->write(\$str);
285 if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) {
287 # Files with read errors (eg: region locked files
288 # on WinXX) are filled with 0 by rsync. Remember
289 # them and delete them later.
292 $badFile =~ s/^\/+//;
293 push(@{$t->{badFiles}}, {
294 share => $t->{shareName},
300 $t->{pidHandler}(@_);
302 fio => BackupPC::Xfer::RsyncFileIO->new({
306 backups => $t->{backups},
307 logLevel => $t->{logLevel}
308 || $conf->{RsyncLogLevel},
312 $t->{XferLOG}->write(\$str);
314 cacheCheckProb => $conf->{RsyncCsumCacheVerifyProb},
319 delete($t->{_errStr});
328 my $conf = $t->{conf};
329 my($remoteSend, $remoteDir, $remoteDirDaemon);
331 alarm($conf->{ClientTimeout});
332 if ( $t->{type} eq "restore" ) {
334 ($remoteDir = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
335 ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
336 $remoteDirDaemon = $t->{shareNameSlash}
337 if ( $t->{pathHdrDest} eq ""
338 || $t->{pathHdrDest} eq "/" );
341 $remoteDir = $t->{shareNameSlash};
342 $remoteDirDaemon = ".";
344 if ( $t->{XferMethod} eq "rsync" ) {
348 my $str = "Running: "
349 . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}})
351 $t->{XferLOG}->write(\$str);
352 $rs->remoteStart($remoteSend, $remoteDir);
355 # Connect to the rsync server
357 if ( defined(my $err = $rs->serverConnect($t->{hostIP},
358 $conf->{RsyncdClientPort})) ) {
359 $t->{hostError} = $err;
360 my $str = "Error connecting to rsync daemon at $t->{hostIP}"
361 . ":$conf->{RsyncdClientPort}: $err\n";
362 $t->{XferLOG}->write(\$str);
366 # Pass module name, and follow it with a slash if it already
367 # contains a slash; otherwise just keep the plain module name.
369 my $module = $t->{shareName};
370 $module = $t->{shareNameSlash} if ( $module =~ /\// );
371 if ( defined(my $err = $rs->serverService($module,
372 $conf->{RsyncdUserName},
373 $conf->{RsyncdPasswd},
374 $conf->{RsyncdAuthRequired})) ) {
375 my $str = "Error connecting to module $module at $t->{hostIP}"
376 . ":$conf->{RsyncdClientPort}: $err\n";
377 $t->{XferLOG}->write(\$str);
378 $t->{hostError} = $err;
381 $rs->serverStart($remoteSend, $remoteDirDaemon);
383 my $error = $rs->go($t->{shareNameSlash});
387 # TODO: generate sensible stats
389 # $rs->{stats}{totalWritten}
390 # $rs->{stats}{totalSize}
392 my $stats = $rs->statsFinal;
393 if ( !defined($error) && defined($stats) ) {
398 $t->{xferErrCnt} = $stats->{remoteErrCnt}
399 + $stats->{childStats}{errorCnt}
400 + $stats->{parentStats}{errorCnt};
401 $t->{byteCnt} = $stats->{childStats}{TotalFileSize}
402 + $stats->{parentStats}{TotalFileSize};
403 $t->{fileCnt} = $stats->{childStats}{TotalFileCnt}
404 + $stats->{parentStats}{TotalFileCnt};
405 my $str = "Done: $t->{fileCnt} files, $t->{byteCnt} bytes\n";
406 $t->{XferLOG}->write(\$str);
408 # TODO: get error count, and call fio to get stats...
410 $t->{hostError} = $error if ( defined($error) );
412 if ( $t->{type} eq "restore" ) {
422 $stats->{childStats}{ExistFileCnt}
423 + $stats->{parentStats}{ExistFileCnt},
424 $stats->{childStats}{ExistFileSize}
425 + $stats->{parentStats}{ExistFileSize},
426 $stats->{childStats}{ExistFileCompSize}
427 + $stats->{parentStats}{ExistFileCompSize},
428 $stats->{childStats}{TotalFileCnt}
429 + $stats->{parentStats}{TotalFileCnt},
430 $stats->{childStats}{TotalFileSize}
431 + $stats->{parentStats}{TotalFileSize},
438 my($t, $reason) = @_;
447 my($t, $FDreadRef) = @_;
454 return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
455 return $t->{_errStr};
469 push(@{$t->{_logMsg}}, $msg);
476 return shift(@{$t->{_logMsg}});
480 # Returns a hash ref giving various status information about
487 return { map { $_ => $t->{$_} }
488 qw(byteCnt fileCnt xferErrCnt xferBadShareCnt xferBadFileCnt
489 xferOK hostAbort hostError lastOutputLine)
497 return @{$t->{badFiles}};