lib/BackupPC/Xfer/Rsync.pm

   1 #============================================================= -*-perl-*-
   2 #
   3 # BackupPC::Xfer::Rsync package
   4 #
   5 # DESCRIPTION
   6 #
   7 #   This library defines a BackupPC::Xfer::Rsync class for managing
   8 #   the rsync-based transport of backup data from the client.
   9 #
  10 # AUTHOR
  11 #   Craig Barratt  <cbarratt@users.sourceforge.net>
  12 #
  13 # COPYRIGHT
  14 #   Copyright (C) 2002-2009  Craig Barratt
  15 #
  16 #   This program is free software; you can redistribute it and/or modify
  17 #   it under the terms of the GNU General Public License as published by
  18 #   the Free Software Foundation; either version 2 of the License, or
  19 #   (at your option) any later version.
  20 #
  21 #   This program is distributed in the hope that it will be useful,
  22 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  23 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  24 #   GNU General Public License for more details.
  25 #
  26 #   You should have received a copy of the GNU General Public License
  27 #   along with this program; if not, write to the Free Software
  28 #   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  29 #
  30 #========================================================================
  31 #
  32 # Version 3.2.0, released 31 Jul 2010.
  33 #
  34 # See http://backuppc.sourceforge.net.
  35 #
  36 #========================================================================
  37
  38 package BackupPC::Xfer::Rsync;
  39
  40 use strict;
  41 use BackupPC::View;
  42 use BackupPC::Xfer::RsyncFileIO;
  43 use Encode qw/from_to encode/;
  44 use base qw(BackupPC::Xfer::Protocol);
  45
  46 use vars qw( $RsyncLibOK $RsyncLibErr );
  47
  48 BEGIN {
  49     eval "use File::RsyncP;";
  50     if ( $@ ) {
  51         #
  52         # Rsync module doesn't exist.
  53         #
  54         $RsyncLibOK = 0;
  55         $RsyncLibErr = "File::RsyncP module doesn't exist";
  56     } else {
  57         #
  58         # Note: also update configure.pl when this version number is changed!
  59         #
  60         if ( $File::RsyncP::VERSION < 0.68 ) {
  61             $RsyncLibOK = 0;
  62             $RsyncLibErr = "File::RsyncP module version"
  63                          . " ($File::RsyncP::VERSION) too old: need >= 0.68";
  64         } else {
  65             $RsyncLibOK = 1;
  66         }
  67     }
  68 };
  69
  70 sub new
  71 {
  72     my($class, $bpc, $args) = @_;
  73
  74     return if ( !$RsyncLibOK );
  75     my $t = BackupPC::Xfer::Protocol->new($bpc, $args);
  76     return bless($t, $class);
  77 }
  78
  79 sub start
  80 {
  81     my($t) = @_;
  82     my $bpc = $t->{bpc};
  83     my $conf = $t->{conf};
  84     my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
  85        $incrDate, $argList, $fioArgs);
  86
  87     #
  88     # We add a slash to the share name we pass to rsync
  89     #
  90     ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{/};
  91
  92     if ( $t->{type} eq "restore" ) {
  93         $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
  94         $rsyncArgs = $conf->{RsyncRestoreArgs};
  95
  96         #
  97         # Merge variables into $rsyncArgs
  98         #
  99         $rsyncArgs = $bpc->cmdVarSubstitute($rsyncArgs, {
 100                             host      => $t->{host},
 101                             hostIP    => $t->{hostIP},
 102                             client    => $t->{client},
 103                             confDir   => $conf->{ConfDir},
 104                         });
 105
 106         my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
 107         $remoteDir    =~ s{//+}{/}g;
 108         from_to($remoteDir, "utf8", $conf->{ClientCharset})
 109                                     if ( $conf->{ClientCharset} ne "" );
 110         $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
 111         $fioArgs = {
 112             client   => $t->{bkupSrcHost},
 113             share    => $t->{bkupSrcShare},
 114             viewNum  => $t->{bkupSrcNum},
 115             fileList => $t->{fileList},
 116         };
 117         $logMsg = "restore started below directory $t->{shareName}"
 118                 . " to host $t->{host}";
 119     } else {
 120         #
 121         # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
 122         # into a hash of arrays of files, and $conf->{RsyncShareName}
 123         # to an array
 124         #
 125         $bpc->backupFileConfFix($conf, "RsyncShareName");
 126
 127         if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
 128             my(@inc, @exc, %incDone, %excDone);
 129             foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
 130                 #
 131                 # If the user wants to just include /home/craig, then
 132                 # we need to do create include/exclude pairs at
 133                 # each level:
 134                 #     --include /home --exclude /*
 135                 #     --include /home/craig --exclude /home/*
 136                 #
 137                 # It's more complex if the user wants to include multiple
 138                 # deep paths.  For example, if they want /home/craig and
 139                 # /var/log, then we need this mouthfull:
 140                 #     --include /home --include /var --exclude /*
 141                 #     --include /home/craig --exclude /home/*
 142                 #     --include /var/log --exclude /var/*
 143                 #
 144                 # To make this easier we do all the includes first and all
 145                 # of the excludes at the end (hopefully they commute).
 146                 #
 147                 $file =~ s{/$}{};
 148                 $file = "/$file";
 149                 $file =~ s{//+}{/}g;
 150                 if ( $file eq "/" ) {
 151                     #
 152                     # This is a special case: if the user specifies
 153                     # "/" then just include it and don't exclude "/*".
 154                     #
 155                     push(@inc, $file) if ( !$incDone{$file} );
 156                     next;
 157                 }
 158                 my $f = "";
 159                 while ( $file =~ m{^/([^/]*)(.*)} ) {
 160                     my $elt = $1;
 161                     $file = $2;
 162                     if ( $file eq "/" ) {
 163                         #
 164                         # preserve a tailing slash
 165                         #
 166                         $file = "";
 167                         $elt = "$elt/";
 168                     }
 169                     push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
 170                     $excDone{"$f/*"} = 1;
 171                     $f = "$f/$elt";
 172                     push(@inc, $f) if ( !$incDone{$f} );
 173                     $incDone{$f} = 1;
 174                 }
 175             }
 176             foreach my $file ( @inc ) {
 177                 $file = encode($conf->{ClientCharset}, $file)
 178                             if ( $conf->{ClientCharset} ne "" );
 179                 push(@fileList, "--include=$file");
 180             }
 181             foreach my $file ( @exc ) {
 182                 $file = encode($conf->{ClientCharset}, $file)
 183                             if ( $conf->{ClientCharset} ne "" );
 184                 push(@fileList, "--exclude=$file");
 185             }
 186         }
 187         if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
 188             foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
 189             {
 190                 #
 191                 # just append additional exclude lists onto the end
 192                 #
 193                 $file = encode($conf->{ClientCharset}, $file)
 194                             if ( $conf->{ClientCharset} ne "" );
 195                 push(@fileList, "--exclude=$file");
 196             }
 197         }
 198         if ( $t->{type} eq "full" ) {
 199             if ( $t->{partialNum} ) {
 200                 $logMsg = "full backup started for directory $t->{shareName};"
 201                         . " updating partial #$t->{partialNum}";
 202             } else {
 203                 $logMsg = "full backup started for directory $t->{shareName}";
 204                 if ( $t->{incrBaseBkupNum} ne "" ) {
 205                     $logMsg .= " (baseline backup #$t->{incrBaseBkupNum})";
 206                 }
 207             }
 208         } else {
 209             $incrDate = $bpc->timeStamp($t->{incrBaseTime}, 1);
 210             $logMsg = "incr backup started back to $incrDate"
 211                     . " (backup #$t->{incrBaseBkupNum}) for directory"
 212                     . " $t->{shareName}";
 213         }
 214
 215         #
 216         # A full dump is implemented with --ignore-times: this causes all
 217         # files to be checksummed, even if the attributes are the same.
 218         # That way all the file contents are checked, but you get all
 219         # the efficiencies of rsync: only files deltas need to be
 220         # transferred, even though it is a full dump.
 221         #
 222         $rsyncArgs = $conf->{RsyncArgs};
 223
 224         #
 225         # Add any additional rsync args
 226         #
 227         $rsyncArgs = [@$rsyncArgs, @{$conf->{RsyncArgsExtra}}]
 228                         if ( ref($conf->{RsyncArgsExtra}) eq 'ARRAY' );
 229
 230         #
 231         # Merge variables into $rsyncArgs
 232         #
 233         $rsyncArgs = $bpc->cmdVarSubstitute($rsyncArgs, {
 234                             host      => $t->{host},
 235                             hostIP    => $t->{hostIP},
 236                             client    => $t->{client},
 237                             confDir   => $conf->{ConfDir},
 238                         });
 239
 240         $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList );
 241         $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
 242                                     if ( $t->{type} eq "full" );
 243         $rsyncClientCmd = $conf->{RsyncClientCmd};
 244         my $shareNameSlash = $t->{shareNameSlash};
 245         from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
 246                             if ( $conf->{ClientCharset} ne "" );
 247         $argList = ['--server', '--sender', @$rsyncArgs,
 248                               '.', $shareNameSlash];
 249         eval {
 250             $argList = File::RsyncP->excludeStrip($argList);
 251         };
 252         $fioArgs = {
 253             client     => $t->{client},
 254             share      => $t->{shareName},
 255             viewNum    => $t->{incrBaseBkupNum},
 256             partialNum => $t->{partialNum},
 257         };
 258     }
 259
 260     #
 261     # Merge variables into $rsyncClientCmd
 262     #
 263     my $args = {
 264         host      => $t->{host},
 265         hostIP    => $t->{hostIP},
 266         client    => $t->{client},
 267         shareName => $t->{shareName},
 268         shareNameSlash => $t->{shareNameSlash},
 269         rsyncPath => $conf->{RsyncClientPath},
 270         sshPath   => $conf->{SshPath},
 271         argList   => $argList,
 272     };
 273     from_to($args->{shareName}, "utf8", $conf->{ClientCharset})
 274                             if ( $conf->{ClientCharset} ne "" );
 275     from_to($args->{shareNameSlash}, "utf8", $conf->{ClientCharset})
 276                             if ( $conf->{ClientCharset} ne "" );
 277     $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args);
 278
 279     #
 280     # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
 281     # module, which handles all the special BackupPC file storage
 282     # (compression, mangling, hardlinks, special files, attributes etc).
 283     #
 284     $t->{rsyncClientCmd} = $rsyncClientCmd;
 285     $t->{rs} = File::RsyncP->new({
 286         logLevel     => $t->{logLevel} || $conf->{RsyncLogLevel},
 287         rsyncCmd     => sub {
 288                             $bpc->verbose(0);
 289                             $bpc->cmdExecOrEval($rsyncClientCmd, $args);
 290                         },
 291         rsyncCmdType => "full",
 292         rsyncArgs    => $rsyncArgs,
 293         timeout      => $conf->{ClientTimeout},
 294         doPartial    => defined($t->{partialNum}) ? 1 : undef,
 295         logHandler   =>
 296                 sub {
 297                     my($str) = @_;
 298                     $str .= "\n";
 299                     $t->{XferLOG}->write(\$str);
 300                     if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) {
 301                         #
 302                         # Files with read errors (eg: region locked files
 303                         # on WinXX) are filled with 0 by rsync.  Remember
 304                         # them and delete them later.
 305                         #
 306                         my $badFile = $1;
 307                         $badFile =~ s/^\/+//;
 308                         push(@{$t->{badFiles}}, {
 309                                 share => $t->{shareName},
 310                                 file  => $badFile
 311                             });
 312                     }
 313                 },
 314         pidHandler   => sub {
 315                             $t->{pidHandler}(@_);
 316                         },
 317         completionPercent => sub {
 318                             $t->{completionPercent}(@_);
 319                         },
 320         clientCharset => $conf->{ClientCharset},
 321         fio          => BackupPC::Xfer::RsyncFileIO->new({
 322                             xfer       => $t,
 323                             bpc        => $t->{bpc},
 324                             conf       => $t->{conf},
 325                             backups    => $t->{backups},
 326                             logLevel   => $t->{logLevel}
 327                                               || $conf->{RsyncLogLevel},
 328                             logHandler => sub {
 329                                               my($str) = @_;
 330                                               $str .= "\n";
 331                                               $t->{XferLOG}->write(\$str);
 332                                           },
 333                             cacheCheckProb => $conf->{RsyncCsumCacheVerifyProb},
 334                             clientCharset  => $conf->{ClientCharset},
 335                             %$fioArgs,
 336                       }),
 337     });
 338
 339     delete($t->{_errStr});
 340
 341     return $logMsg;
 342 }
 343
 344 sub run
 345 {
 346     my($t) = @_;
 347     my $rs = $t->{rs};
 348     my $conf = $t->{conf};
 349     my($remoteSend, $remoteDir, $remoteDirDaemon);
 350
 351     alarm($conf->{ClientTimeout});
 352     if ( $t->{type} eq "restore" ) {
 353         $remoteSend       = 0;
 354         ($remoteDir       = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
 355         ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
 356         $remoteDirDaemon  = $t->{shareNameSlash}
 357                                 if ( $t->{pathHdrDest} eq ""
 358                                               || $t->{pathHdrDest} eq "/" );
 359     } else {
 360         $remoteSend      = 1;
 361         $remoteDir       = $t->{shareNameSlash};
 362         $remoteDirDaemon = ".";
 363     }
 364     from_to($remoteDir, "utf8", $conf->{ClientCharset})
 365                                 if ( $conf->{ClientCharset} ne "" );
 366     from_to($remoteDirDaemon, "utf8", $conf->{ClientCharset})
 367                                 if ( $conf->{ClientCharset} ne "" );
 368
 369     if ( $t->{XferMethod} eq "rsync" ) {
 370         #
 371         # Run rsync command
 372         #
 373         my $str = "Running: "
 374                 . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}})
 375                 . "\n";
 376         from_to($str, $conf->{ClientCharset}, "utf8")
 377                                 if ( $conf->{ClientCharset} ne "" );
 378         $t->{XferLOG}->write(\$str);
 379         $rs->remoteStart($remoteSend, $remoteDir);
 380     } else {
 381         #
 382         # Connect to the rsync server
 383         #
 384         if ( defined(my $err = $rs->serverConnect($t->{hostIP},
 385                                              $conf->{RsyncdClientPort})) ) {
 386             $t->{hostError} = $err;
 387             my $str = "Error connecting to rsync daemon at $t->{hostIP}"
 388                     . ":$conf->{RsyncdClientPort}: $err\n";
 389             $t->{XferLOG}->write(\$str);
 390             return;
 391         }
 392         #
 393         # Pass module name, and follow it with a slash if it already
 394         # contains a slash; otherwise just keep the plain module name.
 395         #
 396         my $module = $t->{shareName};
 397         $module = $t->{shareNameSlash} if ( $module =~ /\// );
 398         from_to($module, "utf8", $conf->{ClientCharset})
 399                                     if ( $conf->{ClientCharset} ne "" );
 400         if ( defined(my $err = $rs->serverService($module,
 401                                              $conf->{RsyncdUserName},
 402                                              $conf->{RsyncdPasswd},
 403                                              $conf->{RsyncdAuthRequired})) ) {
 404             my $str = "Error connecting to module $module at $t->{hostIP}"
 405                     . ":$conf->{RsyncdClientPort}: $err\n";
 406             $t->{XferLOG}->write(\$str);
 407             $t->{hostError} = $err;
 408             return;
 409         }
 410
 411         #
 412         # This is a hack.  To avoid wide chars we encode the arguments
 413         # to utf8 byte streams, then to the client's local charset.
 414         # The second conversion should really go in File::RsyncP, since
 415         # it shouldn't be applied to in-line include/exclude arguments.
 416         #
 417         for ( my $i = 0 ; $i < @{$rs->{rsyncArgs}} ; $i++ ) {
 418             $rs->{rsyncArgs}[$i] = encode('utf8', $rs->{rsyncArgs}[$i]);
 419             from_to($rs->{rsyncArgs}[$i], 'utf8', $conf->{ClientCharset})
 420                                     if ( $conf->{ClientCharset} ne "" );
 421         }
 422
 423         ##my $str = "RsyncArgsBefore: " . join(" ", @{$rs->{rsyncArgs}}) . "\n";
 424         ##$t->{XferLOG}->write(\$str);
 425
 426         $rs->serverStart($remoteSend, $remoteDirDaemon);
 427
 428         ##$str = "RsyncArgsAfter: " . join(" ", @{$rs->{rsyncArgs}}) . "\n";
 429         ##$t->{XferLOG}->write(\$str);
 430     }
 431     my $shareNameSlash = $t->{shareNameSlash};
 432     from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
 433                                 if ( $conf->{ClientCharset} ne "" );
 434     my $error = $rs->go($shareNameSlash);
 435     $rs->serverClose();
 436
 437     #
 438     # TODO: generate sensible stats
 439     #
 440     # $rs->{stats}{totalWritten}
 441     # $rs->{stats}{totalSize}
 442     #
 443     my $stats = $rs->statsFinal;
 444     if ( !defined($error) && defined($stats) ) {
 445         $t->{xferOK} = 1;
 446     } else {
 447         $t->{xferOK} = 0;
 448     }
 449     $t->{xferErrCnt} = $stats->{remoteErrCnt}
 450                      + $stats->{childStats}{errorCnt}
 451                      + $stats->{parentStats}{errorCnt};
 452     $t->{byteCnt}    = $stats->{childStats}{TotalFileSize}
 453                      + $stats->{parentStats}{TotalFileSize};
 454     $t->{fileCnt}    = $stats->{childStats}{TotalFileCnt}
 455                      + $stats->{parentStats}{TotalFileCnt};
 456     my $str = "Done: $t->{fileCnt} files, $t->{byteCnt} bytes\n";
 457     $t->{XferLOG}->write(\$str);
 458     #
 459     # TODO: get error count, and call fio to get stats...
 460     #
 461     $t->{hostError} = $error if ( defined($error) );
 462
 463     if ( $t->{type} eq "restore" ) {
 464         return (
 465             $t->{fileCnt},
 466             $t->{byteCnt},
 467             0,
 468             0
 469         );
 470     } else {
 471         return (
 472             0,
 473             $stats->{childStats}{ExistFileCnt}
 474                 + $stats->{parentStats}{ExistFileCnt},
 475             $stats->{childStats}{ExistFileSize}
 476                 + $stats->{parentStats}{ExistFileSize},
 477             $stats->{childStats}{ExistFileCompSize}
 478                 + $stats->{parentStats}{ExistFileCompSize},
 479             $stats->{childStats}{TotalFileCnt}
 480                 + $stats->{parentStats}{TotalFileCnt},
 481             $stats->{childStats}{TotalFileSize}
 482                 + $stats->{parentStats}{TotalFileSize},
 483         );
 484     }
 485 }
 486
 487 sub abort
 488 {
 489     my($t, $reason) = @_;
 490     my $rs = $t->{rs};
 491
 492     $rs->abort($reason);
 493     return 1;
 494 }
 495
 496 sub errStr
 497 {
 498     my($t) = @_;
 499
 500     return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
 501     return $t->{_errStr};
 502 }
 503
 504 sub xferPid
 505 {
 506     my($t) = @_;
 507
 508     return ();
 509 }
 510
 511 1;