1 #============================================================= -*-perl-*-
3 # BackupPC::Xfer::Rsync package
7 # This library defines a BackupPC::Xfer::Rsync class for managing
8 # the rsync-based transport of backup data from the client.
11 # Craig Barratt <cbarratt@users.sourceforge.net>
14 # Copyright (C) 2002-2007 Craig Barratt
16 # This program is free software; you can redistribute it and/or modify
17 # it under the terms of the GNU General Public License as published by
18 # the Free Software Foundation; either version 2 of the License, or
19 # (at your option) any later version.
21 # This program is distributed in the hope that it will be useful,
22 # but WITHOUT ANY WARRANTY; without even the implied warranty of
23 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24 # GNU General Public License for more details.
26 # You should have received a copy of the GNU General Public License
27 # along with this program; if not, write to the Free Software
28 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
30 #========================================================================
32 # Version 3.2.0, released 31 Dec 2008.
34 # See http://backuppc.sourceforge.net.
36 #========================================================================
38 package BackupPC::Xfer::Rsync;
42 use BackupPC::Xfer::RsyncFileIO;
43 use Encode qw/from_to encode/;
44 use base qw(BackupPC::Xfer::Protocol);
46 use vars qw( $RsyncLibOK $RsyncLibErr );
49 eval "use File::RsyncP;";
52 # Rsync module doesn't exist.
55 $RsyncLibErr = "File::RsyncP module doesn't exist";
58 # Note: also update configure.pl when this version number is changed!
60 if ( $File::RsyncP::VERSION < 0.68 ) {
62 $RsyncLibErr = "File::RsyncP module version"
63 . " ($File::RsyncP::VERSION) too old: need >= 0.68";
72 my($class, $bpc, $args) = @_;
74 return if ( !$RsyncLibOK );
75 my $t = BackupPC::Xfer::Protocol->new($bpc, $args);
76 return bless($t, $class);
83 my $conf = $t->{conf};
84 my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
85 $incrDate, $argList, $fioArgs);
88 # We add a slash to the share name we pass to rsync
90 ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{/};
92 if ( $t->{type} eq "restore" ) {
93 $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
94 $rsyncArgs = $conf->{RsyncRestoreArgs};
95 my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
96 $remoteDir =~ s{//+}{/}g;
97 from_to($remoteDir, "utf8", $conf->{ClientCharset})
98 if ( $conf->{ClientCharset} ne "" );
99 $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
101 client => $t->{bkupSrcHost},
102 share => $t->{bkupSrcShare},
103 viewNum => $t->{bkupSrcNum},
104 fileList => $t->{fileList},
106 $logMsg = "restore started below directory $t->{shareName}"
107 . " to host $t->{host}";
110 # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
111 # into a hash of arrays of files, and $conf->{RsyncShareName}
114 $bpc->backupFileConfFix($conf, "RsyncShareName");
116 if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
117 my(@inc, @exc, %incDone, %excDone);
118 foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
120 # If the user wants to just include /home/craig, then
121 # we need to do create include/exclude pairs at
123 # --include /home --exclude /*
124 # --include /home/craig --exclude /home/*
126 # It's more complex if the user wants to include multiple
127 # deep paths. For example, if they want /home/craig and
128 # /var/log, then we need this mouthfull:
129 # --include /home --include /var --exclude /*
130 # --include /home/craig --exclude /home/*
131 # --include /var/log --exclude /var/*
133 # To make this easier we do all the includes first and all
134 # of the excludes at the end (hopefully they commute).
139 if ( $file eq "/" ) {
141 # This is a special case: if the user specifies
142 # "/" then just include it and don't exclude "/*".
144 push(@inc, $file) if ( !$incDone{$file} );
148 while ( $file =~ m{^/([^/]*)(.*)} ) {
151 if ( $file eq "/" ) {
153 # preserve a tailing slash
158 push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
159 $excDone{"$f/*"} = 1;
161 push(@inc, $f) if ( !$incDone{$f} );
165 foreach my $file ( @inc ) {
166 $file = encode($conf->{ClientCharset}, $file)
167 if ( $conf->{ClientCharset} ne "" );
168 push(@fileList, "--include=$file");
170 foreach my $file ( @exc ) {
171 $file = encode($conf->{ClientCharset}, $file)
172 if ( $conf->{ClientCharset} ne "" );
173 push(@fileList, "--exclude=$file");
176 if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
177 foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
180 # just append additional exclude lists onto the end
182 $file = encode($conf->{ClientCharset}, $file)
183 if ( $conf->{ClientCharset} ne "" );
184 push(@fileList, "--exclude=$file");
187 if ( $t->{type} eq "full" ) {
188 if ( $t->{partialNum} ) {
189 $logMsg = "full backup started for directory $t->{shareName};"
190 . " updating partial #$t->{partialNum}";
192 $logMsg = "full backup started for directory $t->{shareName}";
193 if ( $t->{incrBaseBkupNum} ne "" ) {
194 $logMsg .= " (baseline backup #$t->{incrBaseBkupNum})";
198 $incrDate = $bpc->timeStamp($t->{incrBaseTime}, 1);
199 $logMsg = "incr backup started back to $incrDate"
200 . " (backup #$t->{incrBaseBkupNum}) for directory"
201 . " $t->{shareName}";
205 # A full dump is implemented with --ignore-times: this causes all
206 # files to be checksummed, even if the attributes are the same.
207 # That way all the file contents are checked, but you get all
208 # the efficiencies of rsync: only files deltas need to be
209 # transferred, even though it is a full dump.
211 $rsyncArgs = $conf->{RsyncArgs};
212 $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList );
213 $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
214 if ( $t->{type} eq "full" );
215 $rsyncClientCmd = $conf->{RsyncClientCmd};
216 my $shareNameSlash = $t->{shareNameSlash};
217 from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
218 if ( $conf->{ClientCharset} ne "" );
219 $argList = ['--server', '--sender', @$rsyncArgs,
220 '.', $shareNameSlash];
222 $argList = File::RsyncP->excludeStrip($argList);
225 client => $t->{client},
226 share => $t->{shareName},
227 viewNum => $t->{incrBaseBkupNum},
228 partialNum => $t->{partialNum},
233 # Merge variables into $rsyncClientCmd
237 hostIP => $t->{hostIP},
238 client => $t->{client},
239 shareName => $t->{shareName},
240 shareNameSlash => $t->{shareNameSlash},
241 rsyncPath => $conf->{RsyncClientPath},
242 sshPath => $conf->{SshPath},
245 from_to($args->{shareName}, "utf8", $conf->{ClientCharset})
246 if ( $conf->{ClientCharset} ne "" );
247 from_to($args->{shareNameSlash}, "utf8", $conf->{ClientCharset})
248 if ( $conf->{ClientCharset} ne "" );
249 $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args);
252 # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
253 # module, which handles all the special BackupPC file storage
254 # (compression, mangling, hardlinks, special files, attributes etc).
256 $t->{rsyncClientCmd} = $rsyncClientCmd;
257 $t->{rs} = File::RsyncP->new({
258 logLevel => $t->{logLevel} || $conf->{RsyncLogLevel},
261 $bpc->cmdExecOrEval($rsyncClientCmd, $args);
263 rsyncCmdType => "full",
264 rsyncArgs => $rsyncArgs,
265 timeout => $conf->{ClientTimeout},
266 doPartial => defined($t->{partialNum}) ? 1 : undef,
271 $t->{XferLOG}->write(\$str);
272 if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) {
274 # Files with read errors (eg: region locked files
275 # on WinXX) are filled with 0 by rsync. Remember
276 # them and delete them later.
279 $badFile =~ s/^\/+//;
280 push(@{$t->{badFiles}}, {
281 share => $t->{shareName},
287 $t->{pidHandler}(@_);
289 clientCharset => $conf->{ClientCharset},
290 fio => BackupPC::Xfer::RsyncFileIO->new({
294 backups => $t->{backups},
295 logLevel => $t->{logLevel}
296 || $conf->{RsyncLogLevel},
300 $t->{XferLOG}->write(\$str);
302 cacheCheckProb => $conf->{RsyncCsumCacheVerifyProb},
303 clientCharset => $conf->{ClientCharset},
308 delete($t->{_errStr});
317 my $conf = $t->{conf};
318 my($remoteSend, $remoteDir, $remoteDirDaemon);
320 alarm($conf->{ClientTimeout});
321 if ( $t->{type} eq "restore" ) {
323 ($remoteDir = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
324 ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
325 $remoteDirDaemon = $t->{shareNameSlash}
326 if ( $t->{pathHdrDest} eq ""
327 || $t->{pathHdrDest} eq "/" );
330 $remoteDir = $t->{shareNameSlash};
331 $remoteDirDaemon = ".";
333 from_to($remoteDir, "utf8", $conf->{ClientCharset})
334 if ( $conf->{ClientCharset} ne "" );
335 from_to($remoteDirDaemon, "utf8", $conf->{ClientCharset})
336 if ( $conf->{ClientCharset} ne "" );
338 if ( $t->{XferMethod} eq "rsync" ) {
342 my $str = "Running: "
343 . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}})
345 from_to($str, $conf->{ClientCharset}, "utf8")
346 if ( $conf->{ClientCharset} ne "" );
347 $t->{XferLOG}->write(\$str);
348 $rs->remoteStart($remoteSend, $remoteDir);
351 # Connect to the rsync server
353 if ( defined(my $err = $rs->serverConnect($t->{hostIP},
354 $conf->{RsyncdClientPort})) ) {
355 $t->{hostError} = $err;
356 my $str = "Error connecting to rsync daemon at $t->{hostIP}"
357 . ":$conf->{RsyncdClientPort}: $err\n";
358 $t->{XferLOG}->write(\$str);
362 # Pass module name, and follow it with a slash if it already
363 # contains a slash; otherwise just keep the plain module name.
365 my $module = $t->{shareName};
366 $module = $t->{shareNameSlash} if ( $module =~ /\// );
367 from_to($module, "utf8", $conf->{ClientCharset})
368 if ( $conf->{ClientCharset} ne "" );
369 if ( defined(my $err = $rs->serverService($module,
370 $conf->{RsyncdUserName},
371 $conf->{RsyncdPasswd},
372 $conf->{RsyncdAuthRequired})) ) {
373 my $str = "Error connecting to module $module at $t->{hostIP}"
374 . ":$conf->{RsyncdClientPort}: $err\n";
375 $t->{XferLOG}->write(\$str);
376 $t->{hostError} = $err;
381 # This is a hack. To avoid wide chars we encode the arguments
382 # to utf8 byte streams, then to the client's local charset.
383 # The second conversion should really go in File::RsyncP, since
384 # it shouldn't be applied to in-line include/exclude arguments.
386 for ( my $i = 0 ; $i < @{$rs->{rsyncArgs}} ; $i++ ) {
387 $rs->{rsyncArgs}[$i] = encode('utf8', $rs->{rsyncArgs}[$i]);
388 from_to($rs->{rsyncArgs}[$i], 'utf8', $conf->{ClientCharset})
389 if ( $conf->{ClientCharset} ne "" );
392 my $str = "RsyncArgsBefore: " . join(" ", @{$rs->{rsyncArgs}}) . "\n";
393 $t->{XferLOG}->write(\$str);
395 $rs->serverStart($remoteSend, $remoteDirDaemon);
397 my $str = "RsyncArgsAfter: " . join(" ", @{$rs->{rsyncArgs}}) . "\n";
398 $t->{XferLOG}->write(\$str);
400 my $shareNameSlash = $t->{shareNameSlash};
401 from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
402 if ( $conf->{ClientCharset} ne "" );
403 my $error = $rs->go($shareNameSlash);
407 # TODO: generate sensible stats
409 # $rs->{stats}{totalWritten}
410 # $rs->{stats}{totalSize}
412 my $stats = $rs->statsFinal;
413 if ( !defined($error) && defined($stats) ) {
418 $t->{xferErrCnt} = $stats->{remoteErrCnt}
419 + $stats->{childStats}{errorCnt}
420 + $stats->{parentStats}{errorCnt};
421 $t->{byteCnt} = $stats->{childStats}{TotalFileSize}
422 + $stats->{parentStats}{TotalFileSize};
423 $t->{fileCnt} = $stats->{childStats}{TotalFileCnt}
424 + $stats->{parentStats}{TotalFileCnt};
425 my $str = "Done: $t->{fileCnt} files, $t->{byteCnt} bytes\n";
426 $t->{XferLOG}->write(\$str);
428 # TODO: get error count, and call fio to get stats...
430 $t->{hostError} = $error if ( defined($error) );
432 if ( $t->{type} eq "restore" ) {
442 $stats->{childStats}{ExistFileCnt}
443 + $stats->{parentStats}{ExistFileCnt},
444 $stats->{childStats}{ExistFileSize}
445 + $stats->{parentStats}{ExistFileSize},
446 $stats->{childStats}{ExistFileCompSize}
447 + $stats->{parentStats}{ExistFileCompSize},
448 $stats->{childStats}{TotalFileCnt}
449 + $stats->{parentStats}{TotalFileCnt},
450 $stats->{childStats}{TotalFileSize}
451 + $stats->{parentStats}{TotalFileSize},
458 my($t, $reason) = @_;
469 return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
470 return $t->{_errStr};