# Craig Barratt <cbarratt@users.sourceforge.net>
#
# COPYRIGHT
-# Copyright (C) 2002 Craig Barratt
+# Copyright (C) 2002-2003 Craig Barratt
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
#
#========================================================================
#
-# Version 1.6.0_CVS, released 10 Dec 2002.
+# Version 2.1.0_CVS, released 8 Feb 2004.
#
# See http://backuppc.sourceforge.net.
#
use BackupPC::View;
use BackupPC::Xfer::RsyncFileIO;
-use vars qw( $RsyncLibOK );
+use vars qw( $RsyncLibOK $RsyncLibErr );
BEGIN {
eval "use File::RsyncP;";
# Rsync module doesn't exist.
#
$RsyncLibOK = 0;
+ $RsyncLibErr = "File::RsyncP module doesn't exist";
} else {
- $RsyncLibOK = 1;
+ if ( $File::RsyncP::VERSION < 0.50 ) {
+ $RsyncLibOK = 0;
+ $RsyncLibErr = "File::RsyncP module version too old: need 0.50";
+ } else {
+ $RsyncLibOK = 1;
+ }
}
};
hostIP => "",
shareName => "",
badFiles => [],
+
+ #
+ # Various stats
+ #
+ byteCnt => 0,
+ fileCnt => 0,
+ xferErrCnt => 0,
+ xferBadShareCnt => 0,
+ xferBadFileCnt => 0,
+ xferOK => 0,
+
+ #
+ # User's args
+ #
%$args,
}, $class;
my($t) = @_;
my $bpc = $t->{bpc};
my $conf = $t->{conf};
- my(@fileList, @rsyncClientCmd, $logMsg, $incrDate);
+ my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
+ $incrDate, $argList, $fioArgs);
+
+ #
+ # We add a slash to the share name we pass to rsync
+ #
+ ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{/};
if ( $t->{type} eq "restore" ) {
- # TODO
- #push(@rsyncClientCmd, split(/ +/, $c o n f->{RsyncClientRestoreCmd}));
- $logMsg = "restore not supported for $t->{shareName}";
- #
- # restores are considered to work unless we see they fail
- # (opposite to backups...)
- #
- $t->{xferOK} = 1;
+ $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
+ $rsyncArgs = $conf->{RsyncRestoreArgs};
+ my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
+ $remoteDir =~ s{//+}{/}g;
+ $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
+ $fioArgs = {
+ client => $t->{bkupSrcHost},
+ share => $t->{bkupSrcShare},
+ viewNum => $t->{bkupSrcNum},
+ fileList => $t->{fileList},
+ };
+ $logMsg = "restore started below directory $t->{shareName}"
+ . " to host $t->{host}";
} else {
#
# Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
- # into a hash of arrays of files. NOT IMPLEMENTED YET.
+ # into a hash of arrays of files.
#
$conf->{RsyncShareName} = [ $conf->{RsyncShareName} ]
unless ref($conf->{RsyncShareName}) eq "ARRAY";
};
}
}
+ if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
+ my(@inc, @exc, %incDone, %excDone);
+ foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
+ #
+ # If the user wants to just include /home/craig, then
+ # we need to do create include/exclude pairs at
+ # each level:
+ # --include /home --exclude /*
+ # --include /home/craig --exclude /home/*
+ #
+ # It's more complex if the user wants to include multiple
+ # deep paths. For example, if they want /home/craig and
+ # /var/log, then we need this mouthfull:
+ # --include /home --include /var --exclude /*
+ # --include /home/craig --exclude /home/*
+ # --include /var/log --exclude /var/*
+ #
+ # To make this easier we do all the includes first and all
+ # of the excludes at the end (hopefully they commute).
+ #
+ $file =~ s{/$}{};
+ $file = "/$file";
+ $file =~ s{//+}{/}g;
+ if ( $file eq "/" ) {
+ #
+ # This is a special case: if the user specifies
+ # "/" then just include it and don't exclude "/*".
+ #
+ push(@inc, $file) if ( !$incDone{$file} );
+ next;
+ }
+ my $f = "";
+ while ( $file =~ m{^/([^/]*)(.*)} ) {
+ my $elt = $1;
+ $file = $2;
+ if ( $file eq "/" ) {
+ #
+ # preserve a tailing slash
+ #
+ $file = "";
+ $elt = "$elt/";
+ }
+ push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
+ $excDone{"$f/*"} = 1;
+ $f = "$f/$elt";
+ push(@inc, $f) if ( !$incDone{$f} );
+ $incDone{$f} = 1;
+ }
+ }
+ foreach my $file ( @inc ) {
+ push(@fileList, "--include=$file");
+ }
+ foreach my $file ( @exc ) {
+ push(@fileList, "--exclude=$file");
+ }
+ }
if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
{
+ #
+ # just append additional exclude lists onto the end
+ #
push(@fileList, "--exclude=$file");
}
}
- if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
- foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
- push(@fileList, $file);
- }
- } else {
- push(@fileList, ".");
- }
- push(@rsyncClientCmd, split(/ +/, $conf->{RsyncClientCmd}));
if ( $t->{type} eq "full" ) {
- $logMsg = "full backup started for directory $t->{shareName}";
+ if ( $t->{partialNum} ) {
+ $logMsg = "full backup started for directory $t->{shareName};"
+ . " updating partial $t->{partialNum}";
+ } else {
+ $logMsg = "full backup started for directory $t->{shareName}";
+ }
} else {
- $incrDate = $bpc->timeStampISO($t->{lastFull} - 3600, 1);
+ $incrDate = $bpc->timeStamp($t->{lastFull} - 3600, 1);
$logMsg = "incr backup started back to $incrDate for directory"
. " $t->{shareName}";
}
- $t->{xferOK} = 0;
- }
- #
- # Merge variables into @rsyncClientCmd
- #
- my $vars = {
- host => $t->{host},
- hostIP => $t->{hostIP},
- shareName => $t->{shareName},
- rsyncPath => $conf->{RsyncClientPath},
- sshPath => $conf->{SshPath},
- };
- my @cmd = @rsyncClientCmd;
- @rsyncClientCmd = ();
- foreach my $arg ( @cmd ) {
- next if ( $arg =~ /^\s*$/ );
- if ( $arg =~ /^\$fileList(\+?)/ ) {
- my $esc = $1 eq "+";
- foreach $arg ( @fileList ) {
- $arg = $bpc->shellEscape($arg) if ( $esc );
- push(@rsyncClientCmd, $arg);
- }
- } elsif ( $arg =~ /^\$argList(\+?)/ ) {
- my $esc = $1 eq "+";
- foreach $arg ( (@{$conf->{RsyncArgs}},
- @{$conf->{RsyncClientArgs}}) ) {
- $arg = $bpc->shellEscape($arg) if ( $esc );
- push(@rsyncClientCmd, $arg);
- }
- } else {
- $arg =~ s{\$(\w+)(\+?)}{
- defined($vars->{$1})
- ? ($2 eq "+" ? $bpc->shellEscape($vars->{$1}) : $vars->{$1})
- : "\$$1"
- }eg;
- push(@rsyncClientCmd, $arg);
- }
+
+ #
+ # A full dump is implemented with --ignore-times: this causes all
+ # files to be checksummed, even if the attributes are the same.
+ # That way all the file contents are checked, but you get all
+ # the efficiencies of rsync: only files deltas need to be
+ # transferred, even though it is a full dump.
+ #
+ $rsyncArgs = $conf->{RsyncArgs};
+ $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList );
+ $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
+ if ( $t->{type} eq "full" );
+ $rsyncClientCmd = $conf->{RsyncClientCmd};
+ $argList = ['--server', '--sender', @$rsyncArgs,
+ '.', $t->{shareNameSlash}];
+ $fioArgs = {
+ client => $t->{client},
+ share => $t->{shareName},
+ viewNum => $t->{lastFullBkupNum},
+ partialNum => $t->{partialNum},
+ };
}
#
- # A full dump is implemented with --ignore-times: this causes all
- # files to be checksummed, even if the attributes are the same.
- # That way all the file contents are checked, but you get all
- # the efficiencies of rsync: only files deltas need to be
- # transferred, even though it is a full dump.
+ # Merge variables into $rsyncClientCmd
#
- my $rsyncArgs = $conf->{RsyncArgs};
- $rsyncArgs = [@$rsyncArgs, "--ignore-times"] if ( $t->{type} eq "full" );
+ my $args = {
+ host => $t->{host},
+ hostIP => $t->{hostIP},
+ client => $t->{client},
+ shareName => $t->{shareName},
+ shareNameSlash => $t->{shareNameSlash},
+ rsyncPath => $conf->{RsyncClientPath},
+ sshPath => $conf->{SshPath},
+ argList => $argList,
+ };
+ $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args);
#
# Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
# module, which handles all the special BackupPC file storage
# (compression, mangling, hardlinks, special files, attributes etc).
#
+ $t->{rsyncClientCmd} = $rsyncClientCmd;
$t->{rs} = File::RsyncP->new({
- logLevel => $conf->{RsyncLogLevel},
- rsyncCmd => \@rsyncClientCmd,
- rsyncArgs => $rsyncArgs,
- logHandler => sub {
- my($str) = @_;
- $str .= "\n";
- $t->{XferLOG}->write(\$str);
- },
- fio => BackupPC::Xfer::RsyncFileIO->new({
+ logLevel => $t->{logLevel} || $conf->{RsyncLogLevel},
+ rsyncCmd => sub {
+ $bpc->verbose(0);
+ $bpc->cmdExecOrEval($rsyncClientCmd, $args);
+ },
+ rsyncCmdType => "full",
+ rsyncArgs => $rsyncArgs,
+ timeout => $conf->{ClientTimeout},
+ doPartial => defined($t->{partialNum}) ? 1 : undef,
+ logHandler =>
+ sub {
+ my($str) = @_;
+ $str .= "\n";
+ $t->{XferLOG}->write(\$str);
+ if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) {
+ #
+ # Files with read errors (eg: region locked files
+ # on WinXX) are filled with 0 by rsync. Remember
+ # them and delete them later.
+ #
+ my $badFile = $1;
+ $badFile =~ s/^\/+//;
+ push(@{$t->{badFiles}}, {
+ share => $t->{shareName},
+ file => $badFile
+ });
+ }
+ },
+ pidHandler => sub {
+ $t->{pidHandler}(@_);
+ },
+ fio => BackupPC::Xfer::RsyncFileIO->new({
xfer => $t,
bpc => $t->{bpc},
conf => $t->{conf},
- host => $t->{host},
backups => $t->{backups},
- logLevel => $conf->{RsyncLogLevel},
+ logLevel => $t->{logLevel}
+ || $conf->{RsyncLogLevel},
+ logHandler => sub {
+ my($str) = @_;
+ $str .= "\n";
+ $t->{XferLOG}->write(\$str);
+ },
+ %$fioArgs,
}),
});
- # TODO: alarm($conf->{SmbClientTimeout});
delete($t->{_errStr});
return $logMsg;
my($t) = @_;
my $rs = $t->{rs};
my $conf = $t->{conf};
+ my($remoteSend, $remoteDir, $remoteDirDaemon);
+ alarm($conf->{ClientTimeout});
+ if ( $t->{type} eq "restore" ) {
+ $remoteSend = 0;
+ ($remoteDir = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
+ ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
+ $remoteDirDaemon = $t->{shareNameSlash}
+ if ( $t->{pathHdrDest} eq ""
+ || $t->{pathHdrDest} eq "/" );
+ } else {
+ $remoteSend = 1;
+ $remoteDir = $t->{shareNameSlash};
+ $remoteDirDaemon = ".";
+ }
if ( $t->{XferMethod} eq "rsync" ) {
#
# Run rsync command
#
- $rs->remoteStart(1, $t->{shareName});
+ my $str = "Running: "
+ . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}})
+ . "\n";
+ $t->{XferLOG}->write(\$str);
+ $rs->remoteStart($remoteSend, $remoteDir);
} else {
#
# Connect to the rsync server
if ( defined(my $err = $rs->serverConnect($t->{hostIP},
$conf->{RsyncdClientPort})) ) {
$t->{hostError} = $err;
+ my $str = "Error connecting to rsync daemon at $t->{hostIP}"
+ . ":$conf->{RsyncdClientPort}: $err\n";
+ $t->{XferLOG}->write(\$str);
return;
}
- if ( defined(my $err = $rs->serverService($t->{shareName},
- "craig", "xyz123", 0)) ) {
+ #
+ # Pass module name, and follow it with a slash if it already
+ # contains a slash; otherwise just keep the plain module name.
+ #
+ my $module = $t->{shareName};
+ $module = $t->{shareNameSlash} if ( $module =~ /\// );
+ if ( defined(my $err = $rs->serverService($module,
+ $conf->{RsyncdUserName},
+ $conf->{RsyncdPasswd},
+ $conf->{RsyncdAuthRequired})) ) {
+ my $str = "Error connecting to module $module at $t->{hostIP}"
+ . ":$conf->{RsyncdClientPort}: $err\n";
+ $t->{XferLOG}->write(\$str);
$t->{hostError} = $err;
return;
}
- $rs->serverStart(1, ".");
+ $rs->serverStart($remoteSend, $remoteDirDaemon);
}
- my $error = $rs->go($t->{shareName});
+ my $error = $rs->go($t->{shareNameSlash});
$rs->serverClose();
#
# $rs->{stats}{totalWritten}
# $rs->{stats}{totalSize}
#
- # qw(byteCnt fileCnt xferErrCnt xferBadShareCnt xferBadFileCnt
- # xferOK hostAbort hostError lastOutputLine)
- #
my $stats = $rs->statsFinal;
if ( !defined($error) && defined($stats) ) {
- $t->{xferOK} = 1;
+ $t->{xferOK} = 1;
} else {
- $t->{xferOK} = 0;
+ $t->{xferOK} = 0;
}
- $t->{byteCnt} = $stats->{childStats}{TotalFileSize}
- + $stats->{parentStats}{TotalFileSize};
- $t->{fileCnt} = $stats->{childStats}{TotalFileCnt}
- + $stats->{parentStats}{TotalFileCnt};
+ $t->{xferErrCnt} = $stats->{remoteErrCnt}
+ + $stats->{childStats}{errorCnt}
+ + $stats->{parentStats}{errorCnt};
+ $t->{byteCnt} = $stats->{childStats}{TotalFileSize}
+ + $stats->{parentStats}{TotalFileSize};
+ $t->{fileCnt} = $stats->{childStats}{TotalFileCnt}
+ + $stats->{parentStats}{TotalFileCnt};
+ my $str = "Done: $t->{fileCnt} files, $t->{byteCnt} bytes\n";
+ $t->{XferLOG}->write(\$str);
#
# TODO: get error count, and call fio to get stats...
#
$t->{hostError} = $error if ( defined($error) );
- return (
- 0,
- $stats->{childStats}{ExistFileCnt}
- + $stats->{parentStats}{ExistFileCnt},
- $stats->{childStats}{ExistFileSize}
- + $stats->{parentStats}{ExistFileSize},
- $stats->{childStats}{ExistFileCompSize}
- + $stats->{parentStats}{ExistFileCompSize},
- $stats->{childStats}{TotalFileCnt}
- + $stats->{parentStats}{TotalFileCnt},
- $stats->{childStats}{TotalFileSize}
- + $stats->{parentStats}{TotalFileSize},
- );
+ if ( $t->{type} eq "restore" ) {
+ return (
+ $t->{fileCnt},
+ $t->{byteCnt},
+ 0,
+ 0
+ );
+ } else {
+ return (
+ 0,
+ $stats->{childStats}{ExistFileCnt}
+ + $stats->{parentStats}{ExistFileCnt},
+ $stats->{childStats}{ExistFileSize}
+ + $stats->{parentStats}{ExistFileSize},
+ $stats->{childStats}{ExistFileCompSize}
+ + $stats->{parentStats}{ExistFileCompSize},
+ $stats->{childStats}{TotalFileCnt}
+ + $stats->{parentStats}{TotalFileCnt},
+ $stats->{childStats}{TotalFileSize}
+ + $stats->{parentStats}{TotalFileSize},
+ );
+ }
}
-# alarm($conf->{SmbClientTimeout});
+sub abort
+{
+ my($t, $reason) = @_;
+ my $rs = $t->{rs};
+
+ $rs->abort($reason);
+ return 1;
+}
sub setSelectMask
{
{
my($t) = @_;
+ return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
return $t->{_errStr};
}
{
my($t) = @_;
- return -1;
+ return ();
}
sub logMsg
projects / BackupPC.git / blobdiff