* Fixed encoding of email subject header in bin/BackupPC_sendEmail as
[BackupPC.git] / lib / BackupPC / Xfer / Rsync.pm
1 #============================================================= -*-perl-*-
2 #
3 # BackupPC::Xfer::Rsync package
4 #
5 # DESCRIPTION
6 #
7 #   This library defines a BackupPC::Xfer::Rsync class for managing
8 #   the rsync-based transport of backup data from the client.
9 #
10 # AUTHOR
11 #   Craig Barratt  <cbarratt@users.sourceforge.net>
12 #
13 # COPYRIGHT
14 #   Copyright (C) 2002-2007  Craig Barratt
15 #
16 #   This program is free software; you can redistribute it and/or modify
17 #   it under the terms of the GNU General Public License as published by
18 #   the Free Software Foundation; either version 2 of the License, or
19 #   (at your option) any later version.
20 #
21 #   This program is distributed in the hope that it will be useful,
22 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
23 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
24 #   GNU General Public License for more details.
25 #
26 #   You should have received a copy of the GNU General Public License
27 #   along with this program; if not, write to the Free Software
28 #   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
29 #
30 #========================================================================
31 #
32 # Version 3.1.0, released 25 Nov 2007.
33 #
34 # See http://backuppc.sourceforge.net.
35 #
36 #========================================================================
37
38 package BackupPC::Xfer::Rsync;
39
40 use strict;
41 use BackupPC::View;
42 use BackupPC::Xfer::RsyncFileIO;
43 use Encode qw/from_to encode/;
44 use base qw(BackupPC::Xfer::Protocol);
45
46 use vars qw( $RsyncLibOK $RsyncLibErr );
47
48 BEGIN {
49     eval "use File::RsyncP;";
50     if ( $@ ) {
51         #
52         # Rsync module doesn't exist.
53         #
54         $RsyncLibOK = 0;
55         $RsyncLibErr = "File::RsyncP module doesn't exist";
56     } else {
57         #
58         # Note: also update configure.pl when this version number is changed!
59         #
60         if ( $File::RsyncP::VERSION < 0.68 ) {
61             $RsyncLibOK = 0;
62             $RsyncLibErr = "File::RsyncP module version"
63                          . " ($File::RsyncP::VERSION) too old: need >= 0.68";
64         } else {
65             $RsyncLibOK = 1;
66         }
67     }
68 };
69
70 sub new
71 {
72     my($class, $bpc, $args) = @_;
73
74     return if ( !$RsyncLibOK );
75     my $t = BackupPC::Xfer::Protocol->new($bpc, $args);
76     return bless($t, $class);
77 }
78
79 sub start
80 {
81     my($t) = @_;
82     my $bpc = $t->{bpc};
83     my $conf = $t->{conf};
84     my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
85        $incrDate, $argList, $fioArgs);
86
87     #
88     # We add a slash to the share name we pass to rsync
89     #
90     ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{/};
91
92     if ( $t->{type} eq "restore" ) {
93         $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
94         $rsyncArgs = $conf->{RsyncRestoreArgs};
95         my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
96         $remoteDir    =~ s{//+}{/}g;
97         from_to($remoteDir, "utf8", $conf->{ClientCharset})
98                                     if ( $conf->{ClientCharset} ne "" );
99         $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
100         $fioArgs = {
101             client   => $t->{bkupSrcHost},
102             share    => $t->{bkupSrcShare},
103             viewNum  => $t->{bkupSrcNum},
104             fileList => $t->{fileList},
105         };
106         $logMsg = "restore started below directory $t->{shareName}"
107                 . " to host $t->{host}";
108     } else {
109         #
110         # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
111         # into a hash of arrays of files, and $conf->{RsyncShareName}
112         # to an array
113         #
114         $bpc->backupFileConfFix($conf, "RsyncShareName");
115
116         if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
117             my(@inc, @exc, %incDone, %excDone);
118             foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
119                 #
120                 # If the user wants to just include /home/craig, then
121                 # we need to do create include/exclude pairs at
122                 # each level:
123                 #     --include /home --exclude /*
124                 #     --include /home/craig --exclude /home/*
125                 #
126                 # It's more complex if the user wants to include multiple
127                 # deep paths.  For example, if they want /home/craig and
128                 # /var/log, then we need this mouthfull:
129                 #     --include /home --include /var --exclude /*
130                 #     --include /home/craig --exclude /home/*
131                 #     --include /var/log --exclude /var/*
132                 #
133                 # To make this easier we do all the includes first and all
134                 # of the excludes at the end (hopefully they commute).
135                 #
136                 $file =~ s{/$}{};
137                 $file = "/$file";
138                 $file =~ s{//+}{/}g;
139                 if ( $file eq "/" ) {
140                     #
141                     # This is a special case: if the user specifies
142                     # "/" then just include it and don't exclude "/*".
143                     #
144                     push(@inc, $file) if ( !$incDone{$file} );
145                     next;
146                 }
147                 my $f = "";
148                 while ( $file =~ m{^/([^/]*)(.*)} ) {
149                     my $elt = $1;
150                     $file = $2;
151                     if ( $file eq "/" ) {
152                         #
153                         # preserve a tailing slash
154                         #
155                         $file = "";
156                         $elt = "$elt/";
157                     }
158                     push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
159                     $excDone{"$f/*"} = 1;
160                     $f = "$f/$elt";
161                     push(@inc, $f) if ( !$incDone{$f} );
162                     $incDone{$f} = 1;
163                 }
164             }
165             foreach my $file ( @inc ) {
166                 $file = encode($conf->{ClientCharset}, $file)
167                             if ( $conf->{ClientCharset} ne "" );
168                 push(@fileList, "--include=$file");
169             }
170             foreach my $file ( @exc ) {
171                 $file = encode($conf->{ClientCharset}, $file)
172                             if ( $conf->{ClientCharset} ne "" );
173                 push(@fileList, "--exclude=$file");
174             }
175         }
176         if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
177             foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
178             {
179                 #
180                 # just append additional exclude lists onto the end
181                 #
182                 $file = encode($conf->{ClientCharset}, $file)
183                             if ( $conf->{ClientCharset} ne "" );
184                 push(@fileList, "--exclude=$file");
185             }
186         }
187         if ( $t->{type} eq "full" ) {
188             if ( $t->{partialNum} ) {
189                 $logMsg = "full backup started for directory $t->{shareName};"
190                         . " updating partial #$t->{partialNum}";
191             } else {
192                 $logMsg = "full backup started for directory $t->{shareName}";
193                 if ( $t->{incrBaseBkupNum} ne "" ) {
194                     $logMsg .= " (baseline backup #$t->{incrBaseBkupNum})";
195                 }
196             }
197         } else {
198             $incrDate = $bpc->timeStamp($t->{incrBaseTime}, 1);
199             $logMsg = "incr backup started back to $incrDate"
200                     . " (backup #$t->{incrBaseBkupNum}) for directory"
201                     . " $t->{shareName}";
202         }
203         
204         #
205         # A full dump is implemented with --ignore-times: this causes all
206         # files to be checksummed, even if the attributes are the same.
207         # That way all the file contents are checked, but you get all
208         # the efficiencies of rsync: only files deltas need to be
209         # transferred, even though it is a full dump.
210         #
211         $rsyncArgs = $conf->{RsyncArgs};
212         $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList );
213         $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
214                                     if ( $t->{type} eq "full" );
215         $rsyncClientCmd = $conf->{RsyncClientCmd};
216         my $shareNameSlash = $t->{shareNameSlash};
217         from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
218                             if ( $conf->{ClientCharset} ne "" );
219         $argList = ['--server', '--sender', @$rsyncArgs,
220                               '.', $shareNameSlash];
221         eval {
222             $argList = File::RsyncP->excludeStrip($argList);
223         };
224         $fioArgs = {
225             client     => $t->{client},
226             share      => $t->{shareName},
227             viewNum    => $t->{incrBaseBkupNum},
228             partialNum => $t->{partialNum},
229         };
230     }
231
232     #
233     # Merge variables into $rsyncClientCmd
234     #
235     my $args = {
236         host      => $t->{host},
237         hostIP    => $t->{hostIP},
238         client    => $t->{client},
239         shareName => $t->{shareName},
240         shareNameSlash => $t->{shareNameSlash},
241         rsyncPath => $conf->{RsyncClientPath},
242         sshPath   => $conf->{SshPath},
243         argList   => $argList,
244     };
245     from_to($args->{shareName}, "utf8", $conf->{ClientCharset})
246                             if ( $conf->{ClientCharset} ne "" );
247     from_to($args->{shareNameSlash}, "utf8", $conf->{ClientCharset})
248                             if ( $conf->{ClientCharset} ne "" );
249     $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args);
250
251     #
252     # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
253     # module, which handles all the special BackupPC file storage
254     # (compression, mangling, hardlinks, special files, attributes etc).
255     #
256     $t->{rsyncClientCmd} = $rsyncClientCmd;
257     $t->{rs} = File::RsyncP->new({
258         logLevel     => $t->{logLevel} || $conf->{RsyncLogLevel},
259         rsyncCmd     => sub {
260                             $bpc->verbose(0);
261                             $bpc->cmdExecOrEval($rsyncClientCmd, $args);
262                         },
263         rsyncCmdType => "full",
264         rsyncArgs    => $rsyncArgs,
265         timeout      => $conf->{ClientTimeout},
266         doPartial    => defined($t->{partialNum}) ? 1 : undef,
267         logHandler   =>
268                 sub {
269                     my($str) = @_;
270                     $str .= "\n";
271                     $t->{XferLOG}->write(\$str);
272                     if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) {
273                         #
274                         # Files with read errors (eg: region locked files
275                         # on WinXX) are filled with 0 by rsync.  Remember
276                         # them and delete them later.
277                         #
278                         my $badFile = $1;
279                         $badFile =~ s/^\/+//;
280                         push(@{$t->{badFiles}}, {
281                                 share => $t->{shareName},
282                                 file  => $badFile
283                             });
284                     }
285                 },
286         pidHandler   => sub {
287                             $t->{pidHandler}(@_);
288                         },
289         clientCharset => $conf->{ClientCharset},
290         fio          => BackupPC::Xfer::RsyncFileIO->new({
291                             xfer       => $t,
292                             bpc        => $t->{bpc},
293                             conf       => $t->{conf},
294                             backups    => $t->{backups},
295                             logLevel   => $t->{logLevel}
296                                               || $conf->{RsyncLogLevel},
297                             logHandler => sub {
298                                               my($str) = @_;
299                                               $str .= "\n";
300                                               $t->{XferLOG}->write(\$str);
301                                           },
302                             cacheCheckProb => $conf->{RsyncCsumCacheVerifyProb},
303                             clientCharset  => $conf->{ClientCharset},
304                             %$fioArgs,
305                       }),
306     });
307
308     delete($t->{_errStr});
309
310     return $logMsg;
311 }
312
313 sub run
314 {
315     my($t) = @_;
316     my $rs = $t->{rs};
317     my $conf = $t->{conf};
318     my($remoteSend, $remoteDir, $remoteDirDaemon);
319
320     alarm($conf->{ClientTimeout});
321     if ( $t->{type} eq "restore" ) {
322         $remoteSend       = 0;
323         ($remoteDir       = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
324         ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
325         $remoteDirDaemon  = $t->{shareNameSlash}
326                                 if ( $t->{pathHdrDest} eq ""
327                                               || $t->{pathHdrDest} eq "/" );
328     } else {
329         $remoteSend      = 1;
330         $remoteDir       = $t->{shareNameSlash};
331         $remoteDirDaemon = ".";
332     }
333     from_to($remoteDir, "utf8", $conf->{ClientCharset})
334                                 if ( $conf->{ClientCharset} ne "" );
335     from_to($remoteDirDaemon, "utf8", $conf->{ClientCharset})
336                                 if ( $conf->{ClientCharset} ne "" );
337
338     if ( $t->{XferMethod} eq "rsync" ) {
339         #
340         # Run rsync command
341         #
342         my $str = "Running: "
343                 . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}})
344                 . "\n";
345         from_to($str, $conf->{ClientCharset}, "utf8")
346                                 if ( $conf->{ClientCharset} ne "" );
347         $t->{XferLOG}->write(\$str);
348         $rs->remoteStart($remoteSend, $remoteDir);
349     } else {
350         #
351         # Connect to the rsync server
352         #
353         if ( defined(my $err = $rs->serverConnect($t->{hostIP},
354                                              $conf->{RsyncdClientPort})) ) {
355             $t->{hostError} = $err;
356             my $str = "Error connecting to rsync daemon at $t->{hostIP}"
357                     . ":$conf->{RsyncdClientPort}: $err\n";
358             $t->{XferLOG}->write(\$str);
359             return;
360         }
361         #
362         # Pass module name, and follow it with a slash if it already
363         # contains a slash; otherwise just keep the plain module name.
364         #
365         my $module = $t->{shareName};
366         $module = $t->{shareNameSlash} if ( $module =~ /\// );
367         from_to($module, "utf8", $conf->{ClientCharset})
368                                     if ( $conf->{ClientCharset} ne "" );
369         if ( defined(my $err = $rs->serverService($module,
370                                              $conf->{RsyncdUserName},
371                                              $conf->{RsyncdPasswd},
372                                              $conf->{RsyncdAuthRequired})) ) {
373             my $str = "Error connecting to module $module at $t->{hostIP}"
374                     . ":$conf->{RsyncdClientPort}: $err\n";
375             $t->{XferLOG}->write(\$str);
376             $t->{hostError} = $err;
377             return;
378         }
379         $rs->serverStart($remoteSend, $remoteDirDaemon);
380     }
381     my $shareNameSlash = $t->{shareNameSlash};
382     from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
383                                 if ( $conf->{ClientCharset} ne "" );
384     my $error = $rs->go($shareNameSlash);
385     $rs->serverClose();
386
387     #
388     # TODO: generate sensible stats
389     # 
390     # $rs->{stats}{totalWritten}
391     # $rs->{stats}{totalSize}
392     #
393     my $stats = $rs->statsFinal;
394     if ( !defined($error) && defined($stats) ) {
395         $t->{xferOK} = 1;
396     } else {
397         $t->{xferOK} = 0;
398     }
399     $t->{xferErrCnt} = $stats->{remoteErrCnt}
400                      + $stats->{childStats}{errorCnt}
401                      + $stats->{parentStats}{errorCnt};
402     $t->{byteCnt}    = $stats->{childStats}{TotalFileSize}
403                      + $stats->{parentStats}{TotalFileSize};
404     $t->{fileCnt}    = $stats->{childStats}{TotalFileCnt}
405                      + $stats->{parentStats}{TotalFileCnt};
406     my $str = "Done: $t->{fileCnt} files, $t->{byteCnt} bytes\n";
407     $t->{XferLOG}->write(\$str);
408     #
409     # TODO: get error count, and call fio to get stats...
410     #
411     $t->{hostError} = $error if ( defined($error) );
412
413     if ( $t->{type} eq "restore" ) {
414         return (
415             $t->{fileCnt},
416             $t->{byteCnt},
417             0,
418             0
419         );
420     } else {
421         return (
422             0,
423             $stats->{childStats}{ExistFileCnt}
424                 + $stats->{parentStats}{ExistFileCnt},
425             $stats->{childStats}{ExistFileSize}
426                 + $stats->{parentStats}{ExistFileSize},
427             $stats->{childStats}{ExistFileCompSize}
428                 + $stats->{parentStats}{ExistFileCompSize},
429             $stats->{childStats}{TotalFileCnt}
430                 + $stats->{parentStats}{TotalFileCnt},
431             $stats->{childStats}{TotalFileSize}
432                 + $stats->{parentStats}{TotalFileSize},
433         );
434     }
435 }
436
437 sub abort
438 {
439     my($t, $reason) = @_;
440     my $rs = $t->{rs};
441
442     $rs->abort($reason);
443     return 1;
444 }
445
446 sub errStr
447 {
448     my($t) = @_;
449
450     return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
451     return $t->{_errStr};
452 }
453
454 sub xferPid
455 {
456     my($t) = @_;
457
458     return ();
459 }
460
461 1;