lib/BackupPC/Xfer/Rsync.pm

   1 #============================================================= -*-perl-*-
   2 #
   3 # BackupPC::Xfer::Rsync package
   4 #
   5 # DESCRIPTION
   6 #
   7 #   This library defines a BackupPC::Xfer::Rsync class for managing
   8 #   the rsync-based transport of backup data from the client.
   9 #
  10 # AUTHOR
  11 #   Craig Barratt  <cbarratt@users.sourceforge.net>
  12 #
  13 # COPYRIGHT
  14 #   Copyright (C) 2002-2003  Craig Barratt
  15 #
  16 #   This program is free software; you can redistribute it and/or modify
  17 #   it under the terms of the GNU General Public License as published by
  18 #   the Free Software Foundation; either version 2 of the License, or
  19 #   (at your option) any later version.
  20 #
  21 #   This program is distributed in the hope that it will be useful,
  22 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
  23 #   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  24 #   GNU General Public License for more details.
  25 #
  26 #   You should have received a copy of the GNU General Public License
  27 #   along with this program; if not, write to the Free Software
  28 #   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  29 #
  30 #========================================================================
  31 #
  32 # Version 3.1.0beta0, released 3 Sep 2007.
  33 #
  34 # See http://backuppc.sourceforge.net.
  35 #
  36 #========================================================================
  37
  38 package BackupPC::Xfer::Rsync;
  39
  40 use strict;
  41 use BackupPC::View;
  42 use BackupPC::Xfer::RsyncFileIO;
  43 use Encode qw/from_to encode/;
  44
  45 use vars qw( $RsyncLibOK $RsyncLibErr );
  46
  47 BEGIN {
  48     eval "use File::RsyncP;";
  49     if ( $@ ) {
  50         #
  51         # Rsync module doesn't exist.
  52         #
  53         $RsyncLibOK = 0;
  54         $RsyncLibErr = "File::RsyncP module doesn't exist";
  55     } else {
  56         #
  57         # Note: also update configure.pl when this version number is changed!
  58         #
  59         if ( $File::RsyncP::VERSION < 0.68 ) {
  60             $RsyncLibOK = 0;
  61             $RsyncLibErr = "File::RsyncP module version"
  62                          . " ($File::RsyncP::VERSION) too old: need 0.68";
  63         } else {
  64             $RsyncLibOK = 1;
  65         }
  66     }
  67 };
  68
  69 sub new
  70 {
  71     my($class, $bpc, $args) = @_;
  72
  73     return if ( !$RsyncLibOK );
  74     $args ||= {};
  75     my $t = bless {
  76         bpc       => $bpc,
  77         conf      => { $bpc->Conf },
  78         host      => "",
  79         hostIP    => "",
  80         shareName => "",
  81         badFiles  => [],
  82
  83         #
  84         # Various stats
  85         #
  86         byteCnt         => 0,
  87         fileCnt         => 0,
  88         xferErrCnt      => 0,
  89         xferBadShareCnt => 0,
  90         xferBadFileCnt  => 0,
  91         xferOK          => 0,
  92
  93         #
  94         # User's args
  95         #
  96         %$args,
  97     }, $class;
  98
  99     return $t;
 100 }
 101
 102 sub args
 103 {
 104     my($t, $args) = @_;
 105
 106     foreach my $arg ( keys(%$args) ) {
 107         $t->{$arg} = $args->{$arg};
 108     }
 109 }
 110
 111 sub useTar
 112 {
 113     return 0;
 114 }
 115
 116 sub start
 117 {
 118     my($t) = @_;
 119     my $bpc = $t->{bpc};
 120     my $conf = $t->{conf};
 121     my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
 122        $incrDate, $argList, $fioArgs);
 123
 124     #
 125     # We add a slash to the share name we pass to rsync
 126     #
 127     ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{/};
 128
 129     if ( $t->{type} eq "restore" ) {
 130         $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
 131         $rsyncArgs = $conf->{RsyncRestoreArgs};
 132         my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
 133         $remoteDir    =~ s{//+}{/}g;
 134         from_to($remoteDir, "utf8", $conf->{ClientCharset})
 135                                     if ( $conf->{ClientCharset} ne "" );
 136         $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
 137         $fioArgs = {
 138             client   => $t->{bkupSrcHost},
 139             share    => $t->{bkupSrcShare},
 140             viewNum  => $t->{bkupSrcNum},
 141             fileList => $t->{fileList},
 142         };
 143         $logMsg = "restore started below directory $t->{shareName}"
 144                 . " to host $t->{host}";
 145     } else {
 146         #
 147         # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
 148         # into a hash of arrays of files, and $conf->{RsyncShareName}
 149         # to an array
 150         #
 151         $bpc->backupFileConfFix($conf, "RsyncShareName");
 152
 153         if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
 154             my(@inc, @exc, %incDone, %excDone);
 155             foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
 156                 #
 157                 # If the user wants to just include /home/craig, then
 158                 # we need to do create include/exclude pairs at
 159                 # each level:
 160                 #     --include /home --exclude /*
 161                 #     --include /home/craig --exclude /home/*
 162                 #
 163                 # It's more complex if the user wants to include multiple
 164                 # deep paths.  For example, if they want /home/craig and
 165                 # /var/log, then we need this mouthfull:
 166                 #     --include /home --include /var --exclude /*
 167                 #     --include /home/craig --exclude /home/*
 168                 #     --include /var/log --exclude /var/*
 169                 #
 170                 # To make this easier we do all the includes first and all
 171                 # of the excludes at the end (hopefully they commute).
 172                 #
 173                 $file =~ s{/$}{};
 174                 $file = "/$file";
 175                 $file =~ s{//+}{/}g;
 176                 if ( $file eq "/" ) {
 177                     #
 178                     # This is a special case: if the user specifies
 179                     # "/" then just include it and don't exclude "/*".
 180                     #
 181                     push(@inc, $file) if ( !$incDone{$file} );
 182                     next;
 183                 }
 184                 my $f = "";
 185                 while ( $file =~ m{^/([^/]*)(.*)} ) {
 186                     my $elt = $1;
 187                     $file = $2;
 188                     if ( $file eq "/" ) {
 189                         #
 190                         # preserve a tailing slash
 191                         #
 192                         $file = "";
 193                         $elt = "$elt/";
 194                     }
 195                     push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
 196                     $excDone{"$f/*"} = 1;
 197                     $f = "$f/$elt";
 198                     push(@inc, $f) if ( !$incDone{$f} );
 199                     $incDone{$f} = 1;
 200                 }
 201             }
 202             foreach my $file ( @inc ) {
 203                 $file = encode($conf->{ClientCharset}, $file)
 204                             if ( $conf->{ClientCharset} ne "" );
 205                 push(@fileList, "--include=$file");
 206             }
 207             foreach my $file ( @exc ) {
 208                 $file = encode($conf->{ClientCharset}, $file)
 209                             if ( $conf->{ClientCharset} ne "" );
 210                 push(@fileList, "--exclude=$file");
 211             }
 212         }
 213         if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
 214             foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
 215             {
 216                 #
 217                 # just append additional exclude lists onto the end
 218                 #
 219                 $file = encode($conf->{ClientCharset}, $file)
 220                             if ( $conf->{ClientCharset} ne "" );
 221                 push(@fileList, "--exclude=$file");
 222             }
 223         }
 224         if ( $t->{type} eq "full" ) {
 225             if ( $t->{partialNum} ) {
 226                 $logMsg = "full backup started for directory $t->{shareName};"
 227                         . " updating partial #$t->{partialNum}";
 228             } else {
 229                 $logMsg = "full backup started for directory $t->{shareName}";
 230                 if ( $t->{incrBaseBkupNum} ne "" ) {
 231                     $logMsg .= " (baseline backup #$t->{incrBaseBkupNum})";
 232                 }
 233             }
 234         } else {
 235             $incrDate = $bpc->timeStamp($t->{incrBaseTime}, 1);
 236             $logMsg = "incr backup started back to $incrDate"
 237                     . " (backup #$t->{incrBaseBkupNum}) for directory"
 238                     . " $t->{shareName}";
 239         }
 240
 241         #
 242         # A full dump is implemented with --ignore-times: this causes all
 243         # files to be checksummed, even if the attributes are the same.
 244         # That way all the file contents are checked, but you get all
 245         # the efficiencies of rsync: only files deltas need to be
 246         # transferred, even though it is a full dump.
 247         #
 248         $rsyncArgs = $conf->{RsyncArgs};
 249         $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList );
 250         $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
 251                                     if ( $t->{type} eq "full" );
 252         $rsyncClientCmd = $conf->{RsyncClientCmd};
 253         my $shareNameSlash = $t->{shareNameSlash};
 254         from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
 255                             if ( $conf->{ClientCharset} ne "" );
 256         $argList = ['--server', '--sender', @$rsyncArgs,
 257                               '.', $shareNameSlash];
 258         eval {
 259             $argList = File::RsyncP->excludeStrip($argList);
 260         };
 261         $fioArgs = {
 262             client     => $t->{client},
 263             share      => $t->{shareName},
 264             viewNum    => $t->{incrBaseBkupNum},
 265             partialNum => $t->{partialNum},
 266         };
 267     }
 268
 269     #
 270     # Merge variables into $rsyncClientCmd
 271     #
 272     my $args = {
 273         host      => $t->{host},
 274         hostIP    => $t->{hostIP},
 275         client    => $t->{client},
 276         shareName => $t->{shareName},
 277         shareNameSlash => $t->{shareNameSlash},
 278         rsyncPath => $conf->{RsyncClientPath},
 279         sshPath   => $conf->{SshPath},
 280         argList   => $argList,
 281     };
 282     from_to($args->{shareName}, "utf8", $conf->{ClientCharset})
 283                             if ( $conf->{ClientCharset} ne "" );
 284     from_to($args->{shareNameSlash}, "utf8", $conf->{ClientCharset})
 285                             if ( $conf->{ClientCharset} ne "" );
 286     $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args);
 287
 288     #
 289     # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
 290     # module, which handles all the special BackupPC file storage
 291     # (compression, mangling, hardlinks, special files, attributes etc).
 292     #
 293     $t->{rsyncClientCmd} = $rsyncClientCmd;
 294     $t->{rs} = File::RsyncP->new({
 295         logLevel     => $t->{logLevel} || $conf->{RsyncLogLevel},
 296         rsyncCmd     => sub {
 297                             $bpc->verbose(0);
 298                             $bpc->cmdExecOrEval($rsyncClientCmd, $args);
 299                         },
 300         rsyncCmdType => "full",
 301         rsyncArgs    => $rsyncArgs,
 302         timeout      => $conf->{ClientTimeout},
 303         doPartial    => defined($t->{partialNum}) ? 1 : undef,
 304         logHandler   =>
 305                 sub {
 306                     my($str) = @_;
 307                     $str .= "\n";
 308                     $t->{XferLOG}->write(\$str);
 309                     if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) {
 310                         #
 311                         # Files with read errors (eg: region locked files
 312                         # on WinXX) are filled with 0 by rsync.  Remember
 313                         # them and delete them later.
 314                         #
 315                         my $badFile = $1;
 316                         $badFile =~ s/^\/+//;
 317                         push(@{$t->{badFiles}}, {
 318                                 share => $t->{shareName},
 319                                 file  => $badFile
 320                             });
 321                     }
 322                 },
 323         pidHandler   => sub {
 324                             $t->{pidHandler}(@_);
 325                         },
 326         clientCharset => $conf->{ClientCharset},
 327         fio          => BackupPC::Xfer::RsyncFileIO->new({
 328                             xfer       => $t,
 329                             bpc        => $t->{bpc},
 330                             conf       => $t->{conf},
 331                             backups    => $t->{backups},
 332                             logLevel   => $t->{logLevel}
 333                                               || $conf->{RsyncLogLevel},
 334                             logHandler => sub {
 335                                               my($str) = @_;
 336                                               $str .= "\n";
 337                                               $t->{XferLOG}->write(\$str);
 338                                           },
 339                             cacheCheckProb => $conf->{RsyncCsumCacheVerifyProb},
 340                             clientCharset  => $conf->{ClientCharset},
 341                             %$fioArgs,
 342                       }),
 343     });
 344
 345     delete($t->{_errStr});
 346
 347     return $logMsg;
 348 }
 349
 350 sub run
 351 {
 352     my($t) = @_;
 353     my $rs = $t->{rs};
 354     my $conf = $t->{conf};
 355     my($remoteSend, $remoteDir, $remoteDirDaemon);
 356
 357     alarm($conf->{ClientTimeout});
 358     if ( $t->{type} eq "restore" ) {
 359         $remoteSend       = 0;
 360         ($remoteDir       = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
 361         ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
 362         $remoteDirDaemon  = $t->{shareNameSlash}
 363                                 if ( $t->{pathHdrDest} eq ""
 364                                               || $t->{pathHdrDest} eq "/" );
 365     } else {
 366         $remoteSend      = 1;
 367         $remoteDir       = $t->{shareNameSlash};
 368         $remoteDirDaemon = ".";
 369     }
 370     from_to($remoteDir, "utf8", $conf->{ClientCharset})
 371                                 if ( $conf->{ClientCharset} ne "" );
 372     from_to($remoteDirDaemon, "utf8", $conf->{ClientCharset})
 373                                 if ( $conf->{ClientCharset} ne "" );
 374
 375     if ( $t->{XferMethod} eq "rsync" ) {
 376         #
 377         # Run rsync command
 378         #
 379         my $str = "Running: "
 380                 . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}})
 381                 . "\n";
 382         from_to($str, $conf->{ClientCharset}, "utf8")
 383                                 if ( $conf->{ClientCharset} ne "" );
 384         $t->{XferLOG}->write(\$str);
 385         $rs->remoteStart($remoteSend, $remoteDir);
 386     } else {
 387         #
 388         # Connect to the rsync server
 389         #
 390         if ( defined(my $err = $rs->serverConnect($t->{hostIP},
 391                                              $conf->{RsyncdClientPort})) ) {
 392             $t->{hostError} = $err;
 393             my $str = "Error connecting to rsync daemon at $t->{hostIP}"
 394                     . ":$conf->{RsyncdClientPort}: $err\n";
 395             $t->{XferLOG}->write(\$str);
 396             return;
 397         }
 398         #
 399         # Pass module name, and follow it with a slash if it already
 400         # contains a slash; otherwise just keep the plain module name.
 401         #
 402         my $module = $t->{shareName};
 403         $module = $t->{shareNameSlash} if ( $module =~ /\// );
 404         from_to($module, "utf8", $conf->{ClientCharset})
 405                                     if ( $conf->{ClientCharset} ne "" );
 406         if ( defined(my $err = $rs->serverService($module,
 407                                              $conf->{RsyncdUserName},
 408                                              $conf->{RsyncdPasswd},
 409                                              $conf->{RsyncdAuthRequired})) ) {
 410             my $str = "Error connecting to module $module at $t->{hostIP}"
 411                     . ":$conf->{RsyncdClientPort}: $err\n";
 412             $t->{XferLOG}->write(\$str);
 413             $t->{hostError} = $err;
 414             return;
 415         }
 416         $rs->serverStart($remoteSend, $remoteDirDaemon);
 417     }
 418     my $shareNameSlash = $t->{shareNameSlash};
 419     from_to($shareNameSlash, "utf8", $conf->{ClientCharset})
 420                                 if ( $conf->{ClientCharset} ne "" );
 421     my $error = $rs->go($shareNameSlash);
 422     $rs->serverClose();
 423
 424     #
 425     # TODO: generate sensible stats
 426     #
 427     # $rs->{stats}{totalWritten}
 428     # $rs->{stats}{totalSize}
 429     #
 430     my $stats = $rs->statsFinal;
 431     if ( !defined($error) && defined($stats) ) {
 432         $t->{xferOK} = 1;
 433     } else {
 434         $t->{xferOK} = 0;
 435     }
 436     $t->{xferErrCnt} = $stats->{remoteErrCnt}
 437                      + $stats->{childStats}{errorCnt}
 438                      + $stats->{parentStats}{errorCnt};
 439     $t->{byteCnt}    = $stats->{childStats}{TotalFileSize}
 440                      + $stats->{parentStats}{TotalFileSize};
 441     $t->{fileCnt}    = $stats->{childStats}{TotalFileCnt}
 442                      + $stats->{parentStats}{TotalFileCnt};
 443     my $str = "Done: $t->{fileCnt} files, $t->{byteCnt} bytes\n";
 444     $t->{XferLOG}->write(\$str);
 445     #
 446     # TODO: get error count, and call fio to get stats...
 447     #
 448     $t->{hostError} = $error if ( defined($error) );
 449
 450     if ( $t->{type} eq "restore" ) {
 451         return (
 452             $t->{fileCnt},
 453             $t->{byteCnt},
 454             0,
 455             0
 456         );
 457     } else {
 458         return (
 459             0,
 460             $stats->{childStats}{ExistFileCnt}
 461                 + $stats->{parentStats}{ExistFileCnt},
 462             $stats->{childStats}{ExistFileSize}
 463                 + $stats->{parentStats}{ExistFileSize},
 464             $stats->{childStats}{ExistFileCompSize}
 465                 + $stats->{parentStats}{ExistFileCompSize},
 466             $stats->{childStats}{TotalFileCnt}
 467                 + $stats->{parentStats}{TotalFileCnt},
 468             $stats->{childStats}{TotalFileSize}
 469                 + $stats->{parentStats}{TotalFileSize},
 470         );
 471     }
 472 }
 473
 474 sub abort
 475 {
 476     my($t, $reason) = @_;
 477     my $rs = $t->{rs};
 478
 479     $rs->abort($reason);
 480     return 1;
 481 }
 482
 483 sub setSelectMask
 484 {
 485     my($t, $FDreadRef) = @_;
 486 }
 487
 488 sub errStr
 489 {
 490     my($t) = @_;
 491
 492     return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
 493     return $t->{_errStr};
 494 }
 495
 496 sub xferPid
 497 {
 498     my($t) = @_;
 499
 500     return ();
 501 }
 502
 503 sub logMsg
 504 {
 505     my($t, $msg) = @_;
 506
 507     push(@{$t->{_logMsg}}, $msg);
 508 }
 509
 510 sub logMsgGet
 511 {
 512     my($t) = @_;
 513
 514     return shift(@{$t->{_logMsg}});
 515 }
 516
 517 #
 518 # Returns a hash ref giving various status information about
 519 # the transfer.
 520 #
 521 sub getStats
 522 {
 523     my($t) = @_;
 524
 525     return { map { $_ => $t->{$_} }
 526             qw(byteCnt fileCnt xferErrCnt xferBadShareCnt xferBadFileCnt
 527                xferOK hostAbort hostError lastOutputLine)
 528     };
 529 }
 530
 531 sub getBadFiles
 532 {
 533     my($t) = @_;
 534
 535     return @{$t->{badFiles}};
 536 }
 537
 538 1;