1 #============================================================= -*-perl-*-
3 # BackupPC::Xfer::Rsync package
7 # This library defines a BackupPC::Xfer::Rsync class for managing
8 # the rsync-based transport of backup data from the client.
11 # Craig Barratt <cbarratt@users.sourceforge.net>
14 # Copyright (C) 2002 Craig Barratt
16 # This program is free software; you can redistribute it and/or modify
17 # it under the terms of the GNU General Public License as published by
18 # the Free Software Foundation; either version 2 of the License, or
19 # (at your option) any later version.
21 # This program is distributed in the hope that it will be useful,
22 # but WITHOUT ANY WARRANTY; without even the implied warranty of
23 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24 # GNU General Public License for more details.
26 # You should have received a copy of the GNU General Public License
27 # along with this program; if not, write to the Free Software
28 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
30 #========================================================================
32 # Version 1.6.0_CVS, released 10 Dec 2002.
34 # See http://backuppc.sourceforge.net.
36 #========================================================================
38 package BackupPC::Xfer::Rsync;
42 use BackupPC::Xfer::RsyncFileIO;
44 use vars qw( $RsyncLibOK $RsyncLibErr );
47 eval "use File::RsyncP;";
50 # Rsync module doesn't exist.
53 $RsyncLibErr = "File::RsyncP module doesn't exist";
55 if ( $File::RsyncP::VERSION < 0.20 ) {
57 $RsyncLibErr = "File::RsyncP module version too old: need 0.20";
66 my($class, $bpc, $args) = @_;
68 return if ( !$RsyncLibOK );
72 conf => { $bpc->Conf },
101 foreach my $arg ( keys(%$args) ) {
102 $t->{$arg} = $args->{$arg};
115 my $conf = $t->{conf};
116 my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
117 $incrDate, $argList, $fioArgs);
120 # We add a slash to the share name we pass to rsync
122 ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{};
124 if ( $t->{type} eq "restore" ) {
125 $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
126 $rsyncArgs = $conf->{RsyncRestoreArgs};
127 my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
128 $remoteDir =~ s{//+}{/}g;
129 $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
131 host => $t->{bkupSrcHost},
132 share => $t->{bkupSrcShare},
133 viewNum => $t->{bkupSrcNum},
134 fileList => $t->{fileList},
136 $logMsg = "restore started below directory $t->{shareName}"
137 . " to host $t->{host}";
140 # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
141 # into a hash of arrays of files.
143 $conf->{RsyncShareName} = [ $conf->{RsyncShareName} ]
144 unless ref($conf->{RsyncShareName}) eq "ARRAY";
145 foreach my $param qw(BackupFilesOnly BackupFilesExclude) {
146 next if ( !defined($conf->{$param}) );
147 if ( ref($conf->{$param}) eq "ARRAY" ) {
149 $conf->{RsyncShareName}[0] => $conf->{$param}
151 } elsif ( ref($conf->{$param}) eq "HASH" ) {
155 $conf->{RsyncShareName}[0] => [ $conf->{$param} ]
159 if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
160 my(@inc, @exc, %incDone, %excDone);
161 foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
163 # If the user wants to just include /home/craig, then
164 # we need to do create include/exclude pairs at
166 # --include /home --exclude /*
167 # --include /home/craig --exclude /home/*
169 # It's more complex if the user wants to include multiple
170 # deep paths. For example, if they want /home/craig and
171 # /var/log, then we need this mouthfull:
172 # --include /home --include /var --exclude /*
173 # --include /home/craig --exclude /home/*
174 # --include /var/log --exclude /var/*
176 # To make this easier we do all the includes first and all
177 # of the excludes at the end (hopefully they commute).
182 while ( $file =~ m{^/([^/]*)(.*)} ) {
185 if ( $file eq "/" ) {
187 # preserve a tailing slash
192 push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
193 $excDone{"$f/*"} = 1;
195 push(@inc, $f) if ( !$incDone{$f} );
199 foreach my $file ( @inc ) {
200 push(@fileList, "--include=$file");
202 foreach my $file ( @exc ) {
203 push(@fileList, "--exclude=$file");
206 if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
207 foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
210 # just append additional exclude lists onto the end
212 push(@fileList, "--exclude=$file");
215 if ( $t->{type} eq "full" ) {
216 $logMsg = "full backup started for directory $t->{shareName}";
218 $incrDate = $bpc->timeStampISO($t->{lastFull} - 3600, 1);
219 $logMsg = "incr backup started back to $incrDate for directory"
220 . " $t->{shareName}";
224 # A full dump is implemented with --ignore-times: this causes all
225 # files to be checksummed, even if the attributes are the same.
226 # That way all the file contents are checked, but you get all
227 # the efficiencies of rsync: only files deltas need to be
228 # transferred, even though it is a full dump.
230 $rsyncArgs = $conf->{RsyncArgs};
231 $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
232 if ( $t->{type} eq "full" );
233 $rsyncClientCmd = $conf->{RsyncClientCmd};
234 $argList = ['--server', '--sender', @$rsyncArgs,
235 '.', $t->{shareNameSlash}];
238 share => $t->{shareName},
239 viewNum => $t->{lastFullBkupNum},
244 # Merge variables into $rsyncClientCmd
246 $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd,
249 hostIP => $t->{hostIP},
250 shareName => $t->{shareName},
251 shareNameSlash => $t->{shareNameSlash},
252 rsyncPath => $conf->{RsyncClientPath},
253 sshPath => $conf->{SshPath},
258 # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
259 # module, which handles all the special BackupPC file storage
260 # (compression, mangling, hardlinks, special files, attributes etc).
262 $t->{rsyncClientCmd} = $rsyncClientCmd;
263 $t->{rs} = File::RsyncP->new({
264 logLevel => $conf->{RsyncLogLevel},
266 $bpc->cmdExecOrEval($rsyncClientCmd);
268 rsyncCmdType => "full",
269 rsyncArgs => $rsyncArgs,
273 $t->{XferLOG}->write(\$str);
275 fio => BackupPC::Xfer::RsyncFileIO->new({
279 backups => $t->{backups},
280 logLevel => $conf->{RsyncLogLevel},
281 timeout => $conf->{ClientTimeout},
285 $t->{XferLOG}->write(\$str);
291 delete($t->{_errStr});
300 my $conf = $t->{conf};
301 my($remoteSend, $remoteDir, $remoteDirDaemon);
303 alarm($conf->{ClientTimeout});
304 if ( $t->{type} eq "restore" ) {
306 ($remoteDir = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
307 ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
308 $remoteDirDaemon = $t->{shareNameSlash}
309 if ( $t->{pathHdrDest} eq ""
310 || $t->{pathHdrDest} eq "/" );
313 $remoteDir = $t->{shareNameSlash};
314 $remoteDirDaemon = ".";
316 if ( $t->{XferMethod} eq "rsync" ) {
320 $t->{XferLOG}->write(\"Running: @{$t->{rsyncClientCmd}}\n");
321 $rs->remoteStart($remoteSend, $remoteDir);
324 # Connect to the rsync server
326 if ( defined(my $err = $rs->serverConnect($t->{hostIP},
327 $conf->{RsyncdClientPort})) ) {
328 $t->{hostError} = $err;
332 # Pass module name, and follow it with a slash if it already
333 # contains a slash; otherwise just keep the plain module name.
335 my $module = $t->{shareName};
336 $module = $t->{shareNameSlash} if ( $module =~ /\// );
337 if ( defined(my $err = $rs->serverService($module,
338 $conf->{RsyncdUserName},
339 $conf->{RsyncdPasswd},
340 $conf->{RsyncdAuthRequired})) ) {
341 $t->{hostError} = $err;
344 $rs->serverStart($remoteSend, $remoteDirDaemon);
346 my $error = $rs->go($t->{shareNameSlash});
350 # TODO: generate sensible stats
352 # $rs->{stats}{totalWritten}
353 # $rs->{stats}{totalSize}
355 my $stats = $rs->statsFinal;
356 if ( !defined($error) && defined($stats) ) {
361 $t->{byteCnt} = $stats->{childStats}{TotalFileSize}
362 + $stats->{parentStats}{TotalFileSize};
363 $t->{fileCnt} = $stats->{childStats}{TotalFileCnt}
364 + $stats->{parentStats}{TotalFileCnt};
366 # TODO: get error count, and call fio to get stats...
368 $t->{hostError} = $error if ( defined($error) );
370 if ( $t->{type} eq "restore" ) {
380 $stats->{childStats}{ExistFileCnt}
381 + $stats->{parentStats}{ExistFileCnt},
382 $stats->{childStats}{ExistFileSize}
383 + $stats->{parentStats}{ExistFileSize},
384 $stats->{childStats}{ExistFileCompSize}
385 + $stats->{parentStats}{ExistFileCompSize},
386 $stats->{childStats}{TotalFileCnt}
387 + $stats->{parentStats}{TotalFileCnt},
388 $stats->{childStats}{TotalFileSize}
389 + $stats->{parentStats}{TotalFileSize},
396 my($t, $FDreadRef) = @_;
403 return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
404 return $t->{_errStr};
418 push(@{$t->{_logMsg}}, $msg);
425 return shift(@{$t->{_logMsg}});
429 # Returns a hash ref giving various status information about
436 return { map { $_ => $t->{$_} }
437 qw(byteCnt fileCnt xferErrCnt xferBadShareCnt xferBadFileCnt
438 xferOK hostAbort hostError lastOutputLine)
446 return @{$t->{badFiles}};