X-Git-Url: http://git.rot13.org/?p=BackupPC.git;a=blobdiff_plain;f=lib%2FBackupPC%2FXfer%2FRsync.pm;h=dd4f8abe84e34539e24bec86fd968d8349106b7d;hp=f0350c9a9b292c3a5be81143f2fbd8734bd90df0;hb=380a4961e996464a8c19e39a5339cb3a3bff0580;hpb=74dc9d456332757127d5eda4ce32f29377133fa2 diff --git a/lib/BackupPC/Xfer/Rsync.pm b/lib/BackupPC/Xfer/Rsync.pm index f0350c9..dd4f8ab 100644 --- a/lib/BackupPC/Xfer/Rsync.pm +++ b/lib/BackupPC/Xfer/Rsync.pm @@ -11,7 +11,7 @@ # Craig Barratt # # COPYRIGHT -# Copyright (C) 2002 Craig Barratt +# Copyright (C) 2002-2009 Craig Barratt # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ # #======================================================================== # -# Version 2.0.0beta3, released 1 Jun 2003. +# Version 3.2.0, released 31 Jul 2010. # # See http://backuppc.sourceforge.net. # @@ -40,6 +40,8 @@ package BackupPC::Xfer::Rsync; use strict; use BackupPC::View; use BackupPC::Xfer::RsyncFileIO; +use Encode qw/from_to encode/; +use base qw(BackupPC::Xfer::Protocol); use vars qw( $RsyncLibOK $RsyncLibErr ); @@ -52,9 +54,13 @@ BEGIN { $RsyncLibOK = 0; $RsyncLibErr = "File::RsyncP module doesn't exist"; } else { - if ( $File::RsyncP::VERSION < 0.41 ) { + # + # Note: also update configure.pl when this version number is changed! + # + if ( $File::RsyncP::VERSION < 0.68 ) { $RsyncLibOK = 0; - $RsyncLibErr = "File::RsyncP module version too old: need 0.41"; + $RsyncLibErr = "File::RsyncP module version" + . " ($File::RsyncP::VERSION) too old: need >= 0.68"; } else { $RsyncLibOK = 1; } @@ -66,46 +72,8 @@ sub new my($class, $bpc, $args) = @_; return if ( !$RsyncLibOK ); - $args ||= {}; - my $t = bless { - bpc => $bpc, - conf => { $bpc->Conf }, - host => "", - hostIP => "", - shareName => "", - badFiles => [], - - # - # Various stats - # - byteCnt => 0, - fileCnt => 0, - xferErrCnt => 0, - xferBadShareCnt => 0, - xferBadFileCnt => 0, - xferOK => 0, - - # - # User's args - # - %$args, - }, $class; - - return $t; -} - -sub args -{ - my($t, $args) = @_; - - foreach my $arg ( keys(%$args) ) { - $t->{$arg} = $args->{$arg}; - } -} - -sub useTar -{ - return 0; + my $t = BackupPC::Xfer::Protocol->new($bpc, $args); + return bless($t, $class); } sub start @@ -124,8 +92,21 @@ sub start if ( $t->{type} eq "restore" ) { $rsyncClientCmd = $conf->{RsyncClientRestoreCmd}; $rsyncArgs = $conf->{RsyncRestoreArgs}; + + # + # Merge variables into $rsyncArgs + # + $rsyncArgs = $bpc->cmdVarSubstitute($rsyncArgs, { + host => $t->{host}, + hostIP => $t->{hostIP}, + client => $t->{client}, + confDir => $conf->{ConfDir}, + }); + my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}"; $remoteDir =~ s{//+}{/}g; + from_to($remoteDir, "utf8", $conf->{ClientCharset}) + if ( $conf->{ClientCharset} ne "" ); $argList = ['--server', @$rsyncArgs, '.', $remoteDir]; $fioArgs = { client => $t->{bkupSrcHost}, @@ -138,24 +119,11 @@ sub start } else { # # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude} - # into a hash of arrays of files. + # into a hash of arrays of files, and $conf->{RsyncShareName} + # to an array # - $conf->{RsyncShareName} = [ $conf->{RsyncShareName} ] - unless ref($conf->{RsyncShareName}) eq "ARRAY"; - foreach my $param qw(BackupFilesOnly BackupFilesExclude) { - next if ( !defined($conf->{$param}) ); - if ( ref($conf->{$param}) eq "ARRAY" ) { - $conf->{$param} = { - $conf->{RsyncShareName}[0] => $conf->{$param} - }; - } elsif ( ref($conf->{$param}) eq "HASH" ) { - # do nothing - } else { - $conf->{$param} = { - $conf->{RsyncShareName}[0] => [ $conf->{$param} ] - }; - } - } + $bpc->backupFileConfFix($conf, "RsyncShareName"); + if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) { my(@inc, @exc, %incDone, %excDone); foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) { @@ -176,8 +144,17 @@ sub start # To make this easier we do all the includes first and all # of the excludes at the end (hopefully they commute). # + $file =~ s{/$}{}; $file = "/$file"; $file =~ s{//+}{/}g; + if ( $file eq "/" ) { + # + # This is a special case: if the user specifies + # "/" then just include it and don't exclude "/*". + # + push(@inc, $file) if ( !$incDone{$file} ); + next; + } my $f = ""; while ( $file =~ m{^/([^/]*)(.*)} ) { my $elt = $1; @@ -197,9 +174,13 @@ sub start } } foreach my $file ( @inc ) { + $file = encode($conf->{ClientCharset}, $file) + if ( $conf->{ClientCharset} ne "" ); push(@fileList, "--include=$file"); } foreach my $file ( @exc ) { + $file = encode($conf->{ClientCharset}, $file) + if ( $conf->{ClientCharset} ne "" ); push(@fileList, "--exclude=$file"); } } @@ -209,14 +190,25 @@ sub start # # just append additional exclude lists onto the end # + $file = encode($conf->{ClientCharset}, $file) + if ( $conf->{ClientCharset} ne "" ); push(@fileList, "--exclude=$file"); } } if ( $t->{type} eq "full" ) { - $logMsg = "full backup started for directory $t->{shareName}"; + if ( $t->{partialNum} ) { + $logMsg = "full backup started for directory $t->{shareName};" + . " updating partial #$t->{partialNum}"; + } else { + $logMsg = "full backup started for directory $t->{shareName}"; + if ( $t->{incrBaseBkupNum} ne "" ) { + $logMsg .= " (baseline backup #$t->{incrBaseBkupNum})"; + } + } } else { - $incrDate = $bpc->timeStampISO($t->{lastFull} - 3600, 1); - $logMsg = "incr backup started back to $incrDate for directory" + $incrDate = $bpc->timeStamp($t->{incrBaseTime}, 1); + $logMsg = "incr backup started back to $incrDate" + . " (backup #$t->{incrBaseBkupNum}) for directory" . " $t->{shareName}"; } @@ -228,16 +220,40 @@ sub start # transferred, even though it is a full dump. # $rsyncArgs = $conf->{RsyncArgs}; + + # + # Add any additional rsync args + # + $rsyncArgs = [@$rsyncArgs, @{$conf->{RsyncArgsExtra}}] + if ( ref($conf->{RsyncArgsExtra}) eq 'ARRAY' ); + + # + # Merge variables into $rsyncArgs + # + $rsyncArgs = $bpc->cmdVarSubstitute($rsyncArgs, { + host => $t->{host}, + hostIP => $t->{hostIP}, + client => $t->{client}, + confDir => $conf->{ConfDir}, + }); + $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList ); $rsyncArgs = [@$rsyncArgs, "--ignore-times"] if ( $t->{type} eq "full" ); $rsyncClientCmd = $conf->{RsyncClientCmd}; + my $shareNameSlash = $t->{shareNameSlash}; + from_to($shareNameSlash, "utf8", $conf->{ClientCharset}) + if ( $conf->{ClientCharset} ne "" ); $argList = ['--server', '--sender', @$rsyncArgs, - '.', $t->{shareNameSlash}]; + '.', $shareNameSlash]; + eval { + $argList = File::RsyncP->excludeStrip($argList); + }; $fioArgs = { - client => $t->{client}, - share => $t->{shareName}, - viewNum => $t->{lastFullBkupNum}, + client => $t->{client}, + share => $t->{shareName}, + viewNum => $t->{incrBaseBkupNum}, + partialNum => $t->{partialNum}, }; } @@ -254,6 +270,10 @@ sub start sshPath => $conf->{SshPath}, argList => $argList, }; + from_to($args->{shareName}, "utf8", $conf->{ClientCharset}) + if ( $conf->{ClientCharset} ne "" ); + from_to($args->{shareNameSlash}, "utf8", $conf->{ClientCharset}) + if ( $conf->{ClientCharset} ne "" ); $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args); # @@ -263,7 +283,7 @@ sub start # $t->{rsyncClientCmd} = $rsyncClientCmd; $t->{rs} = File::RsyncP->new({ - logLevel => $conf->{RsyncLogLevel}, + logLevel => $t->{logLevel} || $conf->{RsyncLogLevel}, rsyncCmd => sub { $bpc->verbose(0); $bpc->cmdExecOrEval($rsyncClientCmd, $args); @@ -271,25 +291,47 @@ sub start rsyncCmdType => "full", rsyncArgs => $rsyncArgs, timeout => $conf->{ClientTimeout}, - logHandler => sub { - my($str) = @_; - $str .= "\n"; - $t->{XferLOG}->write(\$str); - }, + doPartial => defined($t->{partialNum}) ? 1 : undef, + logHandler => + sub { + my($str) = @_; + $str .= "\n"; + $t->{XferLOG}->write(\$str); + if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) { + # + # Files with read errors (eg: region locked files + # on WinXX) are filled with 0 by rsync. Remember + # them and delete them later. + # + my $badFile = $1; + $badFile =~ s/^\/+//; + push(@{$t->{badFiles}}, { + share => $t->{shareName}, + file => $badFile + }); + } + }, pidHandler => sub { $t->{pidHandler}(@_); }, + completionPercent => sub { + $t->{completionPercent}(@_); + }, + clientCharset => $conf->{ClientCharset}, fio => BackupPC::Xfer::RsyncFileIO->new({ xfer => $t, bpc => $t->{bpc}, conf => $t->{conf}, backups => $t->{backups}, - logLevel => $conf->{RsyncLogLevel}, + logLevel => $t->{logLevel} + || $conf->{RsyncLogLevel}, logHandler => sub { my($str) = @_; $str .= "\n"; $t->{XferLOG}->write(\$str); }, + cacheCheckProb => $conf->{RsyncCsumCacheVerifyProb}, + clientCharset => $conf->{ClientCharset}, %$fioArgs, }), }); @@ -319,6 +361,11 @@ sub run $remoteDir = $t->{shareNameSlash}; $remoteDirDaemon = "."; } + from_to($remoteDir, "utf8", $conf->{ClientCharset}) + if ( $conf->{ClientCharset} ne "" ); + from_to($remoteDirDaemon, "utf8", $conf->{ClientCharset}) + if ( $conf->{ClientCharset} ne "" ); + if ( $t->{XferMethod} eq "rsync" ) { # # Run rsync command @@ -326,6 +373,8 @@ sub run my $str = "Running: " . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}}) . "\n"; + from_to($str, $conf->{ClientCharset}, "utf8") + if ( $conf->{ClientCharset} ne "" ); $t->{XferLOG}->write(\$str); $rs->remoteStart($remoteSend, $remoteDir); } else { @@ -346,6 +395,8 @@ sub run # my $module = $t->{shareName}; $module = $t->{shareNameSlash} if ( $module =~ /\// ); + from_to($module, "utf8", $conf->{ClientCharset}) + if ( $conf->{ClientCharset} ne "" ); if ( defined(my $err = $rs->serverService($module, $conf->{RsyncdUserName}, $conf->{RsyncdPasswd}, @@ -356,9 +407,31 @@ sub run $t->{hostError} = $err; return; } + + # + # This is a hack. To avoid wide chars we encode the arguments + # to utf8 byte streams, then to the client's local charset. + # The second conversion should really go in File::RsyncP, since + # it shouldn't be applied to in-line include/exclude arguments. + # + for ( my $i = 0 ; $i < @{$rs->{rsyncArgs}} ; $i++ ) { + $rs->{rsyncArgs}[$i] = encode('utf8', $rs->{rsyncArgs}[$i]); + from_to($rs->{rsyncArgs}[$i], 'utf8', $conf->{ClientCharset}) + if ( $conf->{ClientCharset} ne "" ); + } + + ##my $str = "RsyncArgsBefore: " . join(" ", @{$rs->{rsyncArgs}}) . "\n"; + ##$t->{XferLOG}->write(\$str); + $rs->serverStart($remoteSend, $remoteDirDaemon); + + ##$str = "RsyncArgsAfter: " . join(" ", @{$rs->{rsyncArgs}}) . "\n"; + ##$t->{XferLOG}->write(\$str); } - my $error = $rs->go($t->{shareNameSlash}); + my $shareNameSlash = $t->{shareNameSlash}; + from_to($shareNameSlash, "utf8", $conf->{ClientCharset}) + if ( $conf->{ClientCharset} ne "" ); + my $error = $rs->go($shareNameSlash); $rs->serverClose(); # @@ -411,9 +484,13 @@ sub run } } -sub setSelectMask +sub abort { - my($t, $FDreadRef) = @_; + my($t, $reason) = @_; + my $rs = $t->{rs}; + + $rs->abort($reason); + return 1; } sub errStr @@ -431,39 +508,4 @@ sub xferPid return (); } -sub logMsg -{ - my($t, $msg) = @_; - - push(@{$t->{_logMsg}}, $msg); -} - -sub logMsgGet -{ - my($t) = @_; - - return shift(@{$t->{_logMsg}}); -} - -# -# Returns a hash ref giving various status information about -# the transfer. -# -sub getStats -{ - my($t) = @_; - - return { map { $_ => $t->{$_} } - qw(byteCnt fileCnt xferErrCnt xferBadShareCnt xferBadFileCnt - xferOK hostAbort hostError lastOutputLine) - }; -} - -sub getBadFiles -{ - my($t) = @_; - - return @{$t->{badFiles}}; -} - 1;