* Copyright date update
[BackupPC.git] / lib / BackupPC / Xfer / Rsync.pm
1 #============================================================= -*-perl-*-
2 #
3 # BackupPC::Xfer::Rsync package
4 #
5 # DESCRIPTION
6 #
7 #   This library defines a BackupPC::Xfer::Rsync class for managing
8 #   the rsync-based transport of backup data from the client.
9 #
10 # AUTHOR
11 #   Craig Barratt  <cbarratt@users.sourceforge.net>
12 #
13 # COPYRIGHT
14 #   Copyright (C) 2002-2009  Craig Barratt
15 #
16 #   This program is free software; you can redistribute it and/or modify
17 #   it under the terms of the GNU General Public License as published by
18 #   the Free Software Foundation; either version 2 of the License, or
19 #   (at your option) any later version.
20 #
21 #   This program is distributed in the hope that it will be useful,
22 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
23 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
24 #   GNU General Public License for more details.
25 #
26 #   You should have received a copy of the GNU General Public License
27 #   along with this program; if not, write to the Free Software
28 #   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
29 #
30 #========================================================================
31 #
32 # Version 3.2.0beta0, released 5 April 2009.
33 #
34 # See http://backuppc.sourceforge.net.
35 #
36 #========================================================================
37
38 package BackupPC::Xfer::Rsync;
39
40 use strict;
41 use BackupPC::View;
42 use BackupPC::Xfer::RsyncFileIO;
43 use Encode qw/from_to encode/;
44 use base qw(BackupPC::Xfer::Protocol);
45
46 use vars qw( $RsyncLibOK $RsyncLibErr );
47
48 BEGIN {
49     eval "use File::RsyncP;";
50     if ( $@ ) {
51         #
52         # Rsync module doesn't exist.
53         #
54         $RsyncLibOK = 0;
55         $RsyncLibErr = "File::RsyncP module doesn't exist";
56     } else {
57         #
58         # Note: also update configure.pl when this version number is changed!
59         #
60         if ( $File::RsyncP::VERSION < 0.68 ) {
61             $RsyncLibOK = 0;
62             $RsyncLibErr = "File::RsyncP module version"
63                          . " ($File::RsyncP::VERSION) too old: need >= 0.68";
64         } else {
65             $RsyncLibOK = 1;
66         }
67     }
68 };
69
70 sub new
71 {
72     my($class, $bpc, $args) = @_;
73
74     return if ( !$RsyncLibOK );
75     my $t = BackupPC::Xfer::Protocol->new($bpc, $args);
76     return bless($t, $class);
77 }
78
79 sub start
80 {
81     my($t) = @_;
82     my $bpc = $t->{bpc};
83     my $conf = $t->{conf};
84     my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
85        $incrDate, $argList, $fioArgs);
86
87     #
88     # We add a slash to the share name we pass to rsync
89     #
90     ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{/};
91
92     if ( $t->{type} eq "restore" ) {
93         $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
94         $rsyncArgs = $conf->{RsyncRestoreArgs};
95
96         #
97         # Merge variables into $rsyncArgs
98         #
99         $rsyncArgs = $bpc->cmdVarSubstitute($rsyncArgs, {
100                             host      => $t->{host},
101                             hostIP    => $t->{hostIP},
102                             client    => $t->{client},
103                             confDir   => $conf->{ConfDir},
104                         });
105
106         my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
107         $remoteDir    =~ s{//+}{/}g;
108         from_to($remoteDir, "utf8", $conf->{ClientCharset})
109                                     if ( $conf->{ClientCharset} ne "" );
110         $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
111         $fioArgs = {
112             client   => $t->{bkupSrcHost},
113             share    => $t->{bkupSrcShare},
114             viewNum  => $t->{bkupSrcNum},
115             fileList => $t->{fileList},
116         };
117         $logMsg = "restore started below directory $t->{shareName}"
118                 . " to host $t->{host}";
119     } else {
120         #
121         # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
122         # into a hash of arrays of files, and $conf->{RsyncShareName}
123         # to an array
124         #
125         $bpc->backupFileConfFix($conf, "RsyncShareName");
126
127         if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
128             my(@inc, @exc, %incDone, %excDone);
129             foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
130                 #
131                 # If the user wants to just include /home/craig, then
132                 # we need to do create include/exclude pairs at
133                 # each level:
134                 #     --include /home --exclude /*
135                 #     --include /home/craig --exclude /home/*
136                 #
137                 # It's more complex if the user wants to include multiple
138                 # deep paths.  For example, if they want /home/craig and
139                 # /var/log, then we need this mouthfull:
140                 #     --include /home --include /var --exclude /*
141                 #     --include /home/craig --exclude /home/*
142                 #     --include /var/log --exclude /var/*
143                 #
144                 # To make this easier we do all the includes first and all
145                 # of the excludes at the end (hopefully they commute).
146                 #
147                 $file =~ s{/$}{};
148                 $file = "/$file";
149                 $file =~ s{//+}{/}g;
150                 if ( $file eq "/" ) {
151                     #
152                     # This is a special case: if the user specifies
153                     # "/" then just include it and don't exclude "/*".
154                     #
155                     push(@inc, $file) if ( !$incDone{$file} );
156                     next;
157                 }
158                 my $f = "";
159                 while ( $file =~ m{^/([^/]*)(.*)} ) {
160                     my $elt = $1;
161                     $file = $2;
162                     if ( $file eq "/" ) {
163                         #
164                         # preserve a tailing slash
165                         #
166                         $file = "";
167                         $elt = "$elt/";
168                     }
169                     push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
170                     $excDone{"$f/*"} = 1;
171                     $f = "$f/$elt";
172                     push(@inc, $f) if ( !$incDone{$f} );
173                     $incDone{$f} = 1;
174                 }
175             }
176             foreach my $file ( @inc ) {
177                 $file = encode($conf->{ClientCharset}, $file)
178                             if ( $conf->{ClientCharset} ne "" );
179                 push(@fileList, "--include=$file");
180             }
181             foreach my $file ( @exc ) {
182                 $file = encode($conf->{ClientCharset}, $file)
183                             if ( $conf->{ClientCharset} ne "" );
184                 push(@fileList, "--exclude=$file");
185             }
186         }
187         if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
188             foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
189             {
190                 #
191                 # just append additional exclude lists onto the end
192                 #
193                 $file = encode($conf->{ClientCharset}, $file)
194                             if ( $conf->{ClientCharset} ne "" );
195                 push(@fileList, "--exclude=$file");
196             }
197         }
198         if ( $t->{type} eq "full" ) {
199             if ( $t->{partialNum} ) {
200                 $logMsg = "full backup started for directory $t->{shareName};"
201                         . " updating partial #$t->{partialNum}";
202             } else {
203                 $logMsg = "full backup started for directory $t->{shareName}";
204                 if ( $t->{incrBaseBkupNum} ne "" ) {
205                     $logMsg .= " (baseline backup #$t->{incrBaseBkupNum})";
206                 }
207             }
208         } else {
209             $incrDate = $bpc->timeStamp($t->{incrBaseTime}, 1);
210             $logMsg = "incr backup started back to $incrDate"
211                     . " (backup #$t->{incrBaseBkupNum}) for directory"
212                     . " $t->{shareName}";
213         }
214         
215         #
216         # A full dump is implemented with --ignore-times: this causes all
217         # files to be checksummed, even if the attributes are the same.
218         # That way all the file contents are checked, but you get all
219         # the efficiencies of rsync: only files deltas need to be
220         # transferred, even though it is a full dump.
221         #
222         $rsyncArgs = $conf->{RsyncArgs};
223
224         #
225         # Add any additional rsync args
226         #
227         $rsyncArgs = [@$rsyncArgs, @{$conf->{RsyncArgsExtra}}]
228                         if ( ref($conf->{RsyncArgsExtra}) eq 'ARRAY' );
229
230         #
231         # Merge variables into $rsyncArgs
232         #
233         $rsyncArgs = $bpc->cmdVarSubstitute($rsyncArgs, {
234                             host      => $t->{host},
235                             hostIP    => $t->{hostIP},
236                             client    => $t->{client},
237                             confDir   => $conf->{ConfDir},
238                         });
239
240         $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList );
241         $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
242                                     if ( $t->{type} eq "full" );
243         $rsyncClientCmd = $conf->{RsyncClientCmd};
244         my $shareNameSlash = $t->{shareNameSlash};
245         from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
246                             if ( $conf->{ClientCharset} ne "" );
247         $argList = ['--server', '--sender', @$rsyncArgs,
248                               '.', $shareNameSlash];
249         eval {
250             $argList = File::RsyncP->excludeStrip($argList);
251         };
252         $fioArgs = {
253             client     => $t->{client},
254             share      => $t->{shareName},
255             viewNum    => $t->{incrBaseBkupNum},
256             partialNum => $t->{partialNum},
257         };
258     }
259
260     #
261     # Merge variables into $rsyncClientCmd
262     #
263     my $args = {
264         host      => $t->{host},
265         hostIP    => $t->{hostIP},
266         client    => $t->{client},
267         shareName => $t->{shareName},
268         shareNameSlash => $t->{shareNameSlash},
269         rsyncPath => $conf->{RsyncClientPath},
270         sshPath   => $conf->{SshPath},
271         argList   => $argList,
272     };
273     from_to($args->{shareName}, "utf8", $conf->{ClientCharset})
274                             if ( $conf->{ClientCharset} ne "" );
275     from_to($args->{shareNameSlash}, "utf8", $conf->{ClientCharset})
276                             if ( $conf->{ClientCharset} ne "" );
277     $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args);
278
279     #
280     # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
281     # module, which handles all the special BackupPC file storage
282     # (compression, mangling, hardlinks, special files, attributes etc).
283     #
284     $t->{rsyncClientCmd} = $rsyncClientCmd;
285     $t->{rs} = File::RsyncP->new({
286         logLevel     => $t->{logLevel} || $conf->{RsyncLogLevel},
287         rsyncCmd     => sub {
288                             $bpc->verbose(0);
289                             $bpc->cmdExecOrEval($rsyncClientCmd, $args);
290                         },
291         rsyncCmdType => "full",
292         rsyncArgs    => $rsyncArgs,
293         timeout      => $conf->{ClientTimeout},
294         doPartial    => defined($t->{partialNum}) ? 1 : undef,
295         logHandler   =>
296                 sub {
297                     my($str) = @_;
298                     $str .= "\n";
299                     $t->{XferLOG}->write(\$str);
300                     if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) {
301                         #
302                         # Files with read errors (eg: region locked files
303                         # on WinXX) are filled with 0 by rsync.  Remember
304                         # them and delete them later.
305                         #
306                         my $badFile = $1;
307                         $badFile =~ s/^\/+//;
308                         push(@{$t->{badFiles}}, {
309                                 share => $t->{shareName},
310                                 file  => $badFile
311                             });
312                     }
313                 },
314         pidHandler   => sub {
315                             $t->{pidHandler}(@_);
316                         },
317         completionPercent => sub {
318                             $t->{completionPercent}(@_);
319                         },
320         clientCharset => $conf->{ClientCharset},
321         fio          => BackupPC::Xfer::RsyncFileIO->new({
322                             xfer       => $t,
323                             bpc        => $t->{bpc},
324                             conf       => $t->{conf},
325                             backups    => $t->{backups},
326                             logLevel   => $t->{logLevel}
327                                               || $conf->{RsyncLogLevel},
328                             logHandler => sub {
329                                               my($str) = @_;
330                                               $str .= "\n";
331                                               $t->{XferLOG}->write(\$str);
332                                           },
333                             cacheCheckProb => $conf->{RsyncCsumCacheVerifyProb},
334                             clientCharset  => $conf->{ClientCharset},
335                             %$fioArgs,
336                       }),
337     });
338
339     delete($t->{_errStr});
340
341     return $logMsg;
342 }
343
344 sub run
345 {
346     my($t) = @_;
347     my $rs = $t->{rs};
348     my $conf = $t->{conf};
349     my($remoteSend, $remoteDir, $remoteDirDaemon);
350
351     alarm($conf->{ClientTimeout});
352     if ( $t->{type} eq "restore" ) {
353         $remoteSend       = 0;
354         ($remoteDir       = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
355         ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
356         $remoteDirDaemon  = $t->{shareNameSlash}
357                                 if ( $t->{pathHdrDest} eq ""
358                                               || $t->{pathHdrDest} eq "/" );
359     } else {
360         $remoteSend      = 1;
361         $remoteDir       = $t->{shareNameSlash};
362         $remoteDirDaemon = ".";
363     }
364     from_to($remoteDir, "utf8", $conf->{ClientCharset})
365                                 if ( $conf->{ClientCharset} ne "" );
366     from_to($remoteDirDaemon, "utf8", $conf->{ClientCharset})
367                                 if ( $conf->{ClientCharset} ne "" );
368
369     if ( $t->{XferMethod} eq "rsync" ) {
370         #
371         # Run rsync command
372         #
373         my $str = "Running: "
374                 . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}})
375                 . "\n";
376         from_to($str, $conf->{ClientCharset}, "utf8")
377                                 if ( $conf->{ClientCharset} ne "" );
378         $t->{XferLOG}->write(\$str);
379         $rs->remoteStart($remoteSend, $remoteDir);
380     } else {
381         #
382         # Connect to the rsync server
383         #
384         if ( defined(my $err = $rs->serverConnect($t->{hostIP},
385                                              $conf->{RsyncdClientPort})) ) {
386             $t->{hostError} = $err;
387             my $str = "Error connecting to rsync daemon at $t->{hostIP}"
388                     . ":$conf->{RsyncdClientPort}: $err\n";
389             $t->{XferLOG}->write(\$str);
390             return;
391         }
392         #
393         # Pass module name, and follow it with a slash if it already
394         # contains a slash; otherwise just keep the plain module name.
395         #
396         my $module = $t->{shareName};
397         $module = $t->{shareNameSlash} if ( $module =~ /\// );
398         from_to($module, "utf8", $conf->{ClientCharset})
399                                     if ( $conf->{ClientCharset} ne "" );
400         if ( defined(my $err = $rs->serverService($module,
401                                              $conf->{RsyncdUserName},
402                                              $conf->{RsyncdPasswd},
403                                              $conf->{RsyncdAuthRequired})) ) {
404             my $str = "Error connecting to module $module at $t->{hostIP}"
405                     . ":$conf->{RsyncdClientPort}: $err\n";
406             $t->{XferLOG}->write(\$str);
407             $t->{hostError} = $err;
408             return;
409         }
410         
411         #
412         # This is a hack.  To avoid wide chars we encode the arguments
413         # to utf8 byte streams, then to the client's local charset.
414         # The second conversion should really go in File::RsyncP, since
415         # it shouldn't be applied to in-line include/exclude arguments.
416         #
417         for ( my $i = 0 ; $i < @{$rs->{rsyncArgs}} ; $i++ ) {
418             $rs->{rsyncArgs}[$i] = encode('utf8', $rs->{rsyncArgs}[$i]);
419             from_to($rs->{rsyncArgs}[$i], 'utf8', $conf->{ClientCharset})
420                                     if ( $conf->{ClientCharset} ne "" );
421         }
422     
423         ##my $str = "RsyncArgsBefore: " . join(" ", @{$rs->{rsyncArgs}}) . "\n";
424         ##$t->{XferLOG}->write(\$str);
425
426         $rs->serverStart($remoteSend, $remoteDirDaemon);
427
428         ##$str = "RsyncArgsAfter: " . join(" ", @{$rs->{rsyncArgs}}) . "\n";
429         ##$t->{XferLOG}->write(\$str);
430     }
431     my $shareNameSlash = $t->{shareNameSlash};
432     from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
433                                 if ( $conf->{ClientCharset} ne "" );
434     my $error = $rs->go($shareNameSlash);
435     $rs->serverClose();
436
437     #
438     # TODO: generate sensible stats
439     # 
440     # $rs->{stats}{totalWritten}
441     # $rs->{stats}{totalSize}
442     #
443     my $stats = $rs->statsFinal;
444     if ( !defined($error) && defined($stats) ) {
445         $t->{xferOK} = 1;
446     } else {
447         $t->{xferOK} = 0;
448     }
449     $t->{xferErrCnt} = $stats->{remoteErrCnt}
450                      + $stats->{childStats}{errorCnt}
451                      + $stats->{parentStats}{errorCnt};
452     $t->{byteCnt}    = $stats->{childStats}{TotalFileSize}
453                      + $stats->{parentStats}{TotalFileSize};
454     $t->{fileCnt}    = $stats->{childStats}{TotalFileCnt}
455                      + $stats->{parentStats}{TotalFileCnt};
456     my $str = "Done: $t->{fileCnt} files, $t->{byteCnt} bytes\n";
457     $t->{XferLOG}->write(\$str);
458     #
459     # TODO: get error count, and call fio to get stats...
460     #
461     $t->{hostError} = $error if ( defined($error) );
462
463     if ( $t->{type} eq "restore" ) {
464         return (
465             $t->{fileCnt},
466             $t->{byteCnt},
467             0,
468             0
469         );
470     } else {
471         return (
472             0,
473             $stats->{childStats}{ExistFileCnt}
474                 + $stats->{parentStats}{ExistFileCnt},
475             $stats->{childStats}{ExistFileSize}
476                 + $stats->{parentStats}{ExistFileSize},
477             $stats->{childStats}{ExistFileCompSize}
478                 + $stats->{parentStats}{ExistFileCompSize},
479             $stats->{childStats}{TotalFileCnt}
480                 + $stats->{parentStats}{TotalFileCnt},
481             $stats->{childStats}{TotalFileSize}
482                 + $stats->{parentStats}{TotalFileSize},
483         );
484     }
485 }
486
487 sub abort
488 {
489     my($t, $reason) = @_;
490     my $rs = $t->{rs};
491
492     $rs->abort($reason);
493     return 1;
494 }
495
496 sub errStr
497 {
498     my($t) = @_;
499
500     return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
501     return $t->{_errStr};
502 }
503
504 sub xferPid
505 {
506     my($t) = @_;
507
508     return ();
509 }
510
511 1;