3110f8774661dbc308202547c3f0dc47b1f44c01
[BackupPC.git] / lib / BackupPC / Xfer / Rsync.pm
1 #============================================================= -*-perl-*-
2 #
3 # BackupPC::Xfer::Rsync package
4 #
5 # DESCRIPTION
6 #
7 #   This library defines a BackupPC::Xfer::Rsync class for managing
8 #   the rsync-based transport of backup data from the client.
9 #
10 # AUTHOR
11 #   Craig Barratt  <cbarratt@users.sourceforge.net>
12 #
13 # COPYRIGHT
14 #   Copyright (C) 2002-2007  Craig Barratt
15 #
16 #   This program is free software; you can redistribute it and/or modify
17 #   it under the terms of the GNU General Public License as published by
18 #   the Free Software Foundation; either version 2 of the License, or
19 #   (at your option) any later version.
20 #
21 #   This program is distributed in the hope that it will be useful,
22 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
23 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
24 #   GNU General Public License for more details.
25 #
26 #   You should have received a copy of the GNU General Public License
27 #   along with this program; if not, write to the Free Software
28 #   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
29 #
30 #========================================================================
31 #
32 # Version 3.1.0, released 25 Nov 2007.
33 #
34 # See http://backuppc.sourceforge.net.
35 #
36 #========================================================================
37
38 package BackupPC::Xfer::Rsync;
39
40 use strict;
41 use BackupPC::View;
42 use BackupPC::Xfer::RsyncFileIO;
43 use Encode qw/from_to encode/;
44
45 use vars qw( $RsyncLibOK $RsyncLibErr );
46
47 BEGIN {
48     eval "use File::RsyncP;";
49     if ( $@ ) {
50         #
51         # Rsync module doesn't exist.
52         #
53         $RsyncLibOK = 0;
54         $RsyncLibErr = "File::RsyncP module doesn't exist";
55     } else {
56         #
57         # Note: also update configure.pl when this version number is changed!
58         #
59         if ( $File::RsyncP::VERSION < 0.68 ) {
60             $RsyncLibOK = 0;
61             $RsyncLibErr = "File::RsyncP module version"
62                          . " ($File::RsyncP::VERSION) too old: need 0.68";
63         } else {
64             $RsyncLibOK = 1;
65         }
66     }
67 };
68
69 sub new
70 {
71     my($class, $bpc, $args) = @_;
72
73     return if ( !$RsyncLibOK );
74     $args ||= {};
75     my $t = bless {
76         bpc       => $bpc,
77         conf      => { $bpc->Conf },
78         host      => "",
79         hostIP    => "",
80         shareName => "",
81         badFiles  => [],
82
83         #
84         # Various stats
85         #
86         byteCnt         => 0,
87         fileCnt         => 0,
88         xferErrCnt      => 0,
89         xferBadShareCnt => 0,
90         xferBadFileCnt  => 0,
91         xferOK          => 0,
92
93         #
94         # User's args
95         #
96         %$args,
97     }, $class;
98
99     return $t;
100 }
101
102 sub args
103 {
104     my($t, $args) = @_;
105
106     foreach my $arg ( keys(%$args) ) {
107         $t->{$arg} = $args->{$arg};
108     }
109 }
110
111 sub useTar
112 {
113     return 0;
114 }
115
116 sub start
117 {
118     my($t) = @_;
119     my $bpc = $t->{bpc};
120     my $conf = $t->{conf};
121     my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
122        $incrDate, $argList, $fioArgs);
123
124     #
125     # We add a slash to the share name we pass to rsync
126     #
127     ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{/};
128
129     if ( $t->{type} eq "restore" ) {
130         $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
131         $rsyncArgs = $conf->{RsyncRestoreArgs};
132         my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
133         $remoteDir    =~ s{//+}{/}g;
134         from_to($remoteDir, "utf8", $conf->{ClientCharset})
135                                     if ( $conf->{ClientCharset} ne "" );
136         $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
137         $fioArgs = {
138             client   => $t->{bkupSrcHost},
139             share    => $t->{bkupSrcShare},
140             viewNum  => $t->{bkupSrcNum},
141             fileList => $t->{fileList},
142         };
143         $logMsg = "restore started below directory $t->{shareName}"
144                 . " to host $t->{host}";
145     } else {
146         #
147         # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
148         # into a hash of arrays of files, and $conf->{RsyncShareName}
149         # to an array
150         #
151         $bpc->backupFileConfFix($conf, "RsyncShareName");
152
153         if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
154             my(@inc, @exc, %incDone, %excDone);
155             foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
156                 #
157                 # If the user wants to just include /home/craig, then
158                 # we need to do create include/exclude pairs at
159                 # each level:
160                 #     --include /home --exclude /*
161                 #     --include /home/craig --exclude /home/*
162                 #
163                 # It's more complex if the user wants to include multiple
164                 # deep paths.  For example, if they want /home/craig and
165                 # /var/log, then we need this mouthfull:
166                 #     --include /home --include /var --exclude /*
167                 #     --include /home/craig --exclude /home/*
168                 #     --include /var/log --exclude /var/*
169                 #
170                 # To make this easier we do all the includes first and all
171                 # of the excludes at the end (hopefully they commute).
172                 #
173                 $file =~ s{/$}{};
174                 $file = "/$file";
175                 $file =~ s{//+}{/}g;
176                 if ( $file eq "/" ) {
177                     #
178                     # This is a special case: if the user specifies
179                     # "/" then just include it and don't exclude "/*".
180                     #
181                     push(@inc, $file) if ( !$incDone{$file} );
182                     next;
183                 }
184                 my $f = "";
185                 while ( $file =~ m{^/([^/]*)(.*)} ) {
186                     my $elt = $1;
187                     $file = $2;
188                     if ( $file eq "/" ) {
189                         #
190                         # preserve a tailing slash
191                         #
192                         $file = "";
193                         $elt = "$elt/";
194                     }
195                     push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
196                     $excDone{"$f/*"} = 1;
197                     $f = "$f/$elt";
198                     push(@inc, $f) if ( !$incDone{$f} );
199                     $incDone{$f} = 1;
200                 }
201             }
202             foreach my $file ( @inc ) {
203                 $file = encode($conf->{ClientCharset}, $file)
204                             if ( $conf->{ClientCharset} ne "" );
205                 push(@fileList, "--include=$file");
206             }
207             foreach my $file ( @exc ) {
208                 $file = encode($conf->{ClientCharset}, $file)
209                             if ( $conf->{ClientCharset} ne "" );
210                 push(@fileList, "--exclude=$file");
211             }
212         }
213         if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
214             foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
215             {
216                 #
217                 # just append additional exclude lists onto the end
218                 #
219                 $file = encode($conf->{ClientCharset}, $file)
220                             if ( $conf->{ClientCharset} ne "" );
221                 push(@fileList, "--exclude=$file");
222             }
223         }
224         if ( $t->{type} eq "full" ) {
225             if ( $t->{partialNum} ) {
226                 $logMsg = "full backup started for directory $t->{shareName};"
227                         . " updating partial #$t->{partialNum}";
228             } else {
229                 $logMsg = "full backup started for directory $t->{shareName}";
230                 if ( $t->{incrBaseBkupNum} ne "" ) {
231                     $logMsg .= " (baseline backup #$t->{incrBaseBkupNum})";
232                 }
233             }
234         } else {
235             $incrDate = $bpc->timeStamp($t->{incrBaseTime}, 1);
236             $logMsg = "incr backup started back to $incrDate"
237                     . " (backup #$t->{incrBaseBkupNum}) for directory"
238                     . " $t->{shareName}";
239         }
240         
241         #
242         # A full dump is implemented with --ignore-times: this causes all
243         # files to be checksummed, even if the attributes are the same.
244         # That way all the file contents are checked, but you get all
245         # the efficiencies of rsync: only files deltas need to be
246         # transferred, even though it is a full dump.
247         #
248         $rsyncArgs = $conf->{RsyncArgs};
249         $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList );
250         $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
251                                     if ( $t->{type} eq "full" );
252         $rsyncClientCmd = $conf->{RsyncClientCmd};
253         my $shareNameSlash = $t->{shareNameSlash};
254         from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
255                             if ( $conf->{ClientCharset} ne "" );
256         $argList = ['--server', '--sender', @$rsyncArgs,
257                               '.', $shareNameSlash];
258         eval {
259             $argList = File::RsyncP->excludeStrip($argList);
260         };
261         $fioArgs = {
262             client     => $t->{client},
263             share      => $t->{shareName},
264             viewNum    => $t->{incrBaseBkupNum},
265             partialNum => $t->{partialNum},
266         };
267     }
268
269     #
270     # Merge variables into $rsyncClientCmd
271     #
272     my $args = {
273         host      => $t->{host},
274         hostIP    => $t->{hostIP},
275         client    => $t->{client},
276         shareName => $t->{shareName},
277         shareNameSlash => $t->{shareNameSlash},
278         rsyncPath => $conf->{RsyncClientPath},
279         sshPath   => $conf->{SshPath},
280         argList   => $argList,
281     };
282     from_to($args->{shareName}, "utf8", $conf->{ClientCharset})
283                             if ( $conf->{ClientCharset} ne "" );
284     from_to($args->{shareNameSlash}, "utf8", $conf->{ClientCharset})
285                             if ( $conf->{ClientCharset} ne "" );
286     $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args);
287
288     #
289     # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
290     # module, which handles all the special BackupPC file storage
291     # (compression, mangling, hardlinks, special files, attributes etc).
292     #
293     $t->{rsyncClientCmd} = $rsyncClientCmd;
294     $t->{rs} = File::RsyncP->new({
295         logLevel     => $t->{logLevel} || $conf->{RsyncLogLevel},
296         rsyncCmd     => sub {
297                             $bpc->verbose(0);
298                             $bpc->cmdExecOrEval($rsyncClientCmd, $args);
299                         },
300         rsyncCmdType => "full",
301         rsyncArgs    => $rsyncArgs,
302         timeout      => $conf->{ClientTimeout},
303         doPartial    => defined($t->{partialNum}) ? 1 : undef,
304         logHandler   =>
305                 sub {
306                     my($str) = @_;
307                     $str .= "\n";
308                     $t->{XferLOG}->write(\$str);
309                     if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) {
310                         #
311                         # Files with read errors (eg: region locked files
312                         # on WinXX) are filled with 0 by rsync.  Remember
313                         # them and delete them later.
314                         #
315                         my $badFile = $1;
316                         $badFile =~ s/^\/+//;
317                         push(@{$t->{badFiles}}, {
318                                 share => $t->{shareName},
319                                 file  => $badFile
320                             });
321                     }
322                 },
323         pidHandler   => sub {
324                             $t->{pidHandler}(@_);
325                         },
326         clientCharset => $conf->{ClientCharset},
327         fio          => BackupPC::Xfer::RsyncFileIO->new({
328                             xfer       => $t,
329                             bpc        => $t->{bpc},
330                             conf       => $t->{conf},
331                             backups    => $t->{backups},
332                             logLevel   => $t->{logLevel}
333                                               || $conf->{RsyncLogLevel},
334                             logHandler => sub {
335                                               my($str) = @_;
336                                               $str .= "\n";
337                                               $t->{XferLOG}->write(\$str);
338                                           },
339                             cacheCheckProb => $conf->{RsyncCsumCacheVerifyProb},
340                             clientCharset  => $conf->{ClientCharset},
341                             %$fioArgs,
342                       }),
343     });
344
345     delete($t->{_errStr});
346
347     return $logMsg;
348 }
349
350 sub run
351 {
352     my($t) = @_;
353     my $rs = $t->{rs};
354     my $conf = $t->{conf};
355     my($remoteSend, $remoteDir, $remoteDirDaemon);
356
357     alarm($conf->{ClientTimeout});
358     if ( $t->{type} eq "restore" ) {
359         $remoteSend       = 0;
360         ($remoteDir       = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
361         ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
362         $remoteDirDaemon  = $t->{shareNameSlash}
363                                 if ( $t->{pathHdrDest} eq ""
364                                               || $t->{pathHdrDest} eq "/" );
365     } else {
366         $remoteSend      = 1;
367         $remoteDir       = $t->{shareNameSlash};
368         $remoteDirDaemon = ".";
369     }
370     from_to($remoteDir, "utf8", $conf->{ClientCharset})
371                                 if ( $conf->{ClientCharset} ne "" );
372     from_to($remoteDirDaemon, "utf8", $conf->{ClientCharset})
373                                 if ( $conf->{ClientCharset} ne "" );
374
375     if ( $t->{XferMethod} eq "rsync" ) {
376         #
377         # Run rsync command
378         #
379         my $str = "Running: "
380                 . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}})
381                 . "\n";
382         from_to($str, $conf->{ClientCharset}, "utf8")
383                                 if ( $conf->{ClientCharset} ne "" );
384         $t->{XferLOG}->write(\$str);
385         $rs->remoteStart($remoteSend, $remoteDir);
386     } else {
387         #
388         # Connect to the rsync server
389         #
390         if ( defined(my $err = $rs->serverConnect($t->{hostIP},
391                                              $conf->{RsyncdClientPort})) ) {
392             $t->{hostError} = $err;
393             my $str = "Error connecting to rsync daemon at $t->{hostIP}"
394                     . ":$conf->{RsyncdClientPort}: $err\n";
395             $t->{XferLOG}->write(\$str);
396             return;
397         }
398         #
399         # Pass module name, and follow it with a slash if it already
400         # contains a slash; otherwise just keep the plain module name.
401         #
402         my $module = $t->{shareName};
403         $module = $t->{shareNameSlash} if ( $module =~ /\// );
404         from_to($module, "utf8", $conf->{ClientCharset})
405                                     if ( $conf->{ClientCharset} ne "" );
406         if ( defined(my $err = $rs->serverService($module,
407                                              $conf->{RsyncdUserName},
408                                              $conf->{RsyncdPasswd},
409                                              $conf->{RsyncdAuthRequired})) ) {
410             my $str = "Error connecting to module $module at $t->{hostIP}"
411                     . ":$conf->{RsyncdClientPort}: $err\n";
412             $t->{XferLOG}->write(\$str);
413             $t->{hostError} = $err;
414             return;
415         }
416         $rs->serverStart($remoteSend, $remoteDirDaemon);
417     }
418     my $shareNameSlash = $t->{shareNameSlash};
419     from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
420                                 if ( $conf->{ClientCharset} ne "" );
421     my $error = $rs->go($shareNameSlash);
422     $rs->serverClose();
423
424     #
425     # TODO: generate sensible stats
426     # 
427     # $rs->{stats}{totalWritten}
428     # $rs->{stats}{totalSize}
429     #
430     my $stats = $rs->statsFinal;
431     if ( !defined($error) && defined($stats) ) {
432         $t->{xferOK} = 1;
433     } else {
434         $t->{xferOK} = 0;
435     }
436     $t->{xferErrCnt} = $stats->{remoteErrCnt}
437                      + $stats->{childStats}{errorCnt}
438                      + $stats->{parentStats}{errorCnt};
439     $t->{byteCnt}    = $stats->{childStats}{TotalFileSize}
440                      + $stats->{parentStats}{TotalFileSize};
441     $t->{fileCnt}    = $stats->{childStats}{TotalFileCnt}
442                      + $stats->{parentStats}{TotalFileCnt};
443     my $str = "Done: $t->{fileCnt} files, $t->{byteCnt} bytes\n";
444     $t->{XferLOG}->write(\$str);
445     #
446     # TODO: get error count, and call fio to get stats...
447     #
448     $t->{hostError} = $error if ( defined($error) );
449
450     if ( $t->{type} eq "restore" ) {
451         return (
452             $t->{fileCnt},
453             $t->{byteCnt},
454             0,
455             0
456         );
457     } else {
458         return (
459             0,
460             $stats->{childStats}{ExistFileCnt}
461                 + $stats->{parentStats}{ExistFileCnt},
462             $stats->{childStats}{ExistFileSize}
463                 + $stats->{parentStats}{ExistFileSize},
464             $stats->{childStats}{ExistFileCompSize}
465                 + $stats->{parentStats}{ExistFileCompSize},
466             $stats->{childStats}{TotalFileCnt}
467                 + $stats->{parentStats}{TotalFileCnt},
468             $stats->{childStats}{TotalFileSize}
469                 + $stats->{parentStats}{TotalFileSize},
470         );
471     }
472 }
473
474 sub abort
475 {
476     my($t, $reason) = @_;
477     my $rs = $t->{rs};
478
479     $rs->abort($reason);
480     return 1;
481 }
482
483 sub setSelectMask
484 {
485     my($t, $FDreadRef) = @_;
486 }
487
488 sub errStr
489 {
490     my($t) = @_;
491
492     return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
493     return $t->{_errStr};
494 }
495
496 sub xferPid
497 {
498     my($t) = @_;
499
500     return ();
501 }
502
503 sub logMsg
504 {
505     my($t, $msg) = @_;
506
507     push(@{$t->{_logMsg}}, $msg);
508 }
509
510 sub logMsgGet
511 {
512     my($t) = @_;
513
514     return shift(@{$t->{_logMsg}});
515 }
516
517 #
518 # Returns a hash ref giving various status information about
519 # the transfer.
520 #
521 sub getStats
522 {
523     my($t) = @_;
524
525     return { map { $_ => $t->{$_} }
526             qw(byteCnt fileCnt xferErrCnt xferBadShareCnt xferBadFileCnt
527                xferOK hostAbort hostError lastOutputLine)
528     };
529 }
530
531 sub getBadFiles
532 {
533     my($t) = @_;
534
535     return @{$t->{badFiles}};
536 }
537
538 1;