From 3dc33e5f39430031766adf3c5bb2ffc649ee9100 Mon Sep 17 00:00:00 2001 From: cbarratt Date: Sun, 19 Jan 2003 17:08:19 +0000 Subject: [PATCH] * Completed support for rsync and rsyncd, including restore. * Added optional user-defined pre/post dump/restore commands, allowing things like database shutdown/startup for dumps * Replaced $Conf{PingArgs} with $Conf{PingCmd}, added $Conf{DfCmd}, $Conf{NmbLookupCmd} allowing all these commands to be fully configured. Also, all commands (except smbclient) can also now be fragments of perl code. --- ChangeLog | 22 +- bin/BackupPC | 6 +- bin/BackupPC_dump | 65 ++++- bin/BackupPC_nightly | 6 +- bin/BackupPC_restore | 392 +++++++++++++++++++---------- cgi-bin/BackupPC_Admin | 29 ++- conf/config.pl | 179 +++++++++++--- configure.pl | 50 +++- doc-src/BackupPC.pod | 192 ++++++++++++-- lib/BackupPC/Lang/en.pm | 4 +- lib/BackupPC/Lang/fr.pm | 4 +- lib/BackupPC/Lib.pm | 413 ++++++++++++++++++++----------- lib/BackupPC/View.pm | 39 ++- lib/BackupPC/Xfer/Rsync.pm | 286 ++++++++++++++------- lib/BackupPC/Xfer/RsyncFileIO.pm | 242 +++++++++++++----- lib/BackupPC/Xfer/Smb.pm | 4 +- lib/BackupPC/Xfer/Tar.pm | 59 ++--- makeDist | 97 +++++++- 18 files changed, 1503 insertions(+), 586 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4a6f7f1..49f9915 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,18 +21,28 @@ # Version __VERSION__, __RELEASEDATE__ #------------------------------------------------------------------------ -* Support for rsync and rsyncd. Changes to BackupPC_dump and new - modules BackupPC::Xfer::Rsync and BackupPC::Xfer::RsyncFileIO. +* Support for rsync and rsyncd backup and restore. Changes to + BackupPC_dump, BackupPC_restore, and new modules BackupPC::Xfer::Rsync + and BackupPC::Xfer::RsyncFileIO. + +* Added internationalization (i18n) code from Xavier Nicollet, + with additions from Guillaume Filion. Voila! BackupPC_Admin + now supports English and French, and adding more languages is + now easy. + +* Added optional user-defined pre/post dump/restore commands, allowing + things like database shutdown/startup for dumps. + +* Replaced $Conf{PingArgs} with $Conf{PingCmd}, added $Conf{DfCmd}, + $Conf{NmbLookupCmd} allowing all these commands to be fully + configured. Also, all commands (except smbclient) can also + now be fragments of perl code. * Added new BackupPC::View module that creates views of backups (handling merging etc). Updated BackupPC_Admin, BackupPC_zipCreate and BackupPC_tarCreate to use BackupPC::View. This removes lots of merging and mangling code from the higher-level code. -* Added internationalization (i18n) code from Xavier Nicollet. - Voila! BackupPC_Admin now supports English and French, and - adding more languages is now easy. - * Added patch from Toby Johnson that allows additional users to be specified in the hosts file; these users can also view/start/stop and restore backups for that host. Also added a new config diff --git a/bin/BackupPC b/bin/BackupPC index ab70f88..964640e 100755 --- a/bin/BackupPC +++ b/bin/BackupPC @@ -882,6 +882,7 @@ sub Main_Check_Job_Messages $Info{"$f[0]FileCntRep"} = $f[7]; $Info{"$f[0]FileRepMax"} = $f[8]; $Info{"$f[0]FileCntRename"} = $f[9]; + $Info{"$f[0]FileLinkMax"} = $f[10]; $Info{"$f[0]Time"} = time; printf(LOG "%s%s nightly clean removed %d files of" . " size %.2fGB\n", @@ -889,11 +890,12 @@ sub Main_Check_Job_Messages $Info{"$f[0]FileCntRm"}, $Info{"$f[0]KbRm"} / (1000 * 1024)); printf(LOG "%s%s is %.2fGB, %d files (%d repeated, " - . "%d max chain), %d directories\n", + . "%d max chain, %d max links), %d directories\n", $bpc->timeStamp, ucfirst($f[0]), $Info{"$f[0]Kb"} / (1000 * 1024), $Info{"$f[0]FileCnt"}, $Info{"$f[0]FileCntRep"}, - $Info{"$f[0]FileRepMax"}, $Info{"$f[0]DirCnt"}); + $Info{"$f[0]FileRepMax"}, + $Info{"$f[0]FileLinkMax"}, $Info{"$f[0]DirCnt"}); } elsif ( $mesg =~ /^BackupPC_nightly lock_off/ ) { $RunNightlyWhenIdle = 0; } elsif ( $mesg =~ /^processState\s+(.+)/ ) { diff --git a/bin/BackupPC_dump b/bin/BackupPC_dump index 7fe11fe..fcb0bd7 100755 --- a/bin/BackupPC_dump +++ b/bin/BackupPC_dump @@ -91,6 +91,7 @@ die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); my $TopDir = $bpc->TopDir(); my $BinDir = $bpc->BinDir(); my %Conf = $bpc->Conf(); +my $NeedPostCmd; $bpc->ChildInit(); @@ -146,7 +147,7 @@ $SIG{TERM} = \&catch_signal; # Make sure we eventually timeout if there is no activity from # the data transport program. # -alarm($Conf{SmbClientTimeout}); +alarm($Conf{ClientTimeout}); mkpath($Dir, 0, 0777) if ( !-d $Dir ); if ( !-f "$Dir/LOCK" ) { @@ -331,6 +332,12 @@ if ( $Conf{XferMethod} eq "tar" ) { $ShareNames = [ $ShareNames ] unless ref($ShareNames) eq "ARRAY"; +# +# Run an optional pre-dump command +# +UserCommandRun("DumpPreUserCmd"); +$NeedPostCmd = 1; + # # Now backup each of the shares # @@ -355,9 +362,10 @@ for my $shareName ( @$ShareNames ) { # Use rsync as the transport program. # if ( !defined($xfer = BackupPC::Xfer::Rsync->new($bpc)) ) { - print(LOG $bpc->timeStamp, - "dump failed: File::RsyncP module is not installed\n"); - print("dump failed: Rsync module is not installed\n"); + my $errStr = BackupPC::Xfer::Rsync::errStr; + print(LOG $bpc->timeStamp, "dump failed: $errStr\n"); + print("dump failed: $errStr\n"); + UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); exit(1); } } else { @@ -366,6 +374,7 @@ for my $shareName ( @$ShareNames ) { # $xfer = BackupPC::Xfer::Smb->new($bpc); } + my $useTar = $xfer->useTar; if ( $useTar ) { @@ -438,7 +447,7 @@ for my $shareName ( @$ShareNames ) { lastFullBkupNum => $lastFullBkupNum, backups => \@Backups, compress => $Conf{CompressLevel}, - XferMethod => => $Conf{XferMethod}, + XferMethod => $Conf{XferMethod}, }); if ( !defined($logMsg = $xfer->start()) ) { @@ -453,6 +462,7 @@ for my $shareName ( @$ShareNames ) { sleep(1); kill(9, $tarPid); } + UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); exit(1); } @@ -577,9 +587,6 @@ for my $shareName ( @$ShareNames ) { last; } } -$XferLOG->close(); -close($newFilesFH) if ( defined($newFilesFH) ); - my $lastNum = -1; # @@ -589,6 +596,11 @@ if ( $stat{xferOK} && (my $errMsg = CorrectHostCheck($hostIP, $host)) ) { $stat{hostError} = $errMsg; $stat{xferOK} = 0; } + +UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); +$XferLOG->close(); +close($newFilesFH) if ( defined($newFilesFH) ); + if ( $stat{xferOK} ) { @Backups = $bpc->BackupInfoRead($host); for ( my $i = 0 ; $i < @Backups ; $i++ ) { @@ -723,6 +735,7 @@ sub catch_signal my $fileExt = $Conf{CompressLevel} > 0 ? ".z" : ""; print(LOG $bpc->timeStamp, "cleaning up after signal $signame\n"); + UserCommandRun("DumpPostUserCmd") if ( $NeedPostCmd ); $XferLOG->write(\"exiting after signal $signame\n"); $XferLOG->close(); if ( $xferPid > 0 ) { @@ -846,3 +859,39 @@ sub CorrectHostCheck if ( $netBiosHost ne $host ); return; } + +# +# Run an optional pre- or post-dump command +# +sub UserCommandRun +{ + my($type) = @_; + + return if ( !defined($Conf{$type}) ); + my $vars = { + xfer => $xfer, + host => $host, + hostIP => $hostIP, + share => $ShareNames->[0], + shares => $ShareNames, + XferMethod => $Conf{XferMethod}, + LOG => *LOG, + XferLOG => $XferLOG, + stat => \%stat, + xferOK => $stat{xferOK}, + type => $type, + }; + my $cmd = $bpc->cmdVarSubstitute($Conf{$type}, $vars); + $XferLOG->write(\"Executing $type: @$cmd\n"); + # + # Run the user's command, dumping the stdout/stderr into the + # Xfer log file. Also supply the optional $vars and %Conf in + # case the command is really perl code instead of a shell + # command. + # + $bpc->cmdSystemOrEval($cmd, + sub { + $XferLOG->write(\$_[0]); + }, + $vars, \%Conf); +} diff --git a/bin/BackupPC_nightly b/bin/BackupPC_nightly index b1a06ae..1f2d1f4 100755 --- a/bin/BackupPC_nightly +++ b/bin/BackupPC_nightly @@ -120,6 +120,7 @@ my $fileCntRep; # total number of file names containing "_", ie: files # that have repeated md5 checksums my $fileRepMax; # worse case number of files that have repeated checksums # (ie: max(nnn+1) for all names xxxxxxxxxxxxxxxx_nnn) +my $fileLinkMax; # maximum number of hardlinks on a pool file my $fileCntRename; # number of renamed files (to keep file numbering # contiguous) my %FixList; # list of paths that need to be renamed to avoid @@ -133,6 +134,7 @@ for my $pool ( qw(pool cpool) ) { $blkCnt2 = 0; $fileCntRep = 0; $fileRepMax = 0; + $fileLinkMax = 0; $fileCntRename = 0; %FixList = (); find({wanted => \&GetPoolStats, no_chdir => 1}, "$TopDir/$pool"); @@ -168,7 +170,8 @@ for my $pool ( qw(pool cpool) ) { } } print("BackupPC_stats = $pool,$fileCnt,$dirCnt,$kb,$kb2,$kbRm,$fileCntRm," - . "$fileCntRep,$fileRepMax,$fileCntRename\n"); + . "$fileCntRep,$fileRepMax,$fileCntRename," + . "$fileLinkMax\n"); } ########################################################################### @@ -218,5 +221,6 @@ sub GetPoolStats $fileCnt += -f; $blkCnt += $s[12]; $blkCnt2 += $s[12] if ( -f && $s[3] == 2 ); + $fileLinkMax = $s[3] if ( $fileLinkMax < $s[3] ); } } diff --git a/bin/BackupPC_restore b/bin/BackupPC_restore index 5104dd7..c01f12c 100755 --- a/bin/BackupPC_restore +++ b/bin/BackupPC_restore @@ -41,6 +41,7 @@ use BackupPC::Lib; use BackupPC::FileZIO; use BackupPC::Xfer::Smb; use BackupPC::Xfer::Tar; +use BackupPC::Xfer::Rsync; use File::Path; use Getopt::Std; @@ -55,6 +56,7 @@ die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); my $TopDir = $bpc->TopDir(); my $BinDir = $bpc->BinDir(); my %Conf = $bpc->Conf(); +my $NeedPostCmd; my($hostIP, $host, $reqFileName); @@ -77,7 +79,10 @@ my $Hosts = $bpc->HostInfoRead(); # # Re-read config file, so we can include the PC-specific config # -$bpc->ConfigRead($host); +if ( defined(my $error = $bpc->ConfigRead($host)) ) { + print("Can't read PC's config file: $error\n"); + exit(1); +} %Conf = $bpc->Conf(); my $Dir = "$TopDir/pc/$host"; @@ -104,7 +109,7 @@ if ( !(my $ret = do "$Dir/$reqFileName") ) { # Make sure we eventually timeout if there is no activity from # the data transport program. # -alarm($Conf{SmbClientTimeout}); +alarm($Conf{ClientTimeout}); mkpath($Dir, 0, 0777) if ( !-d $Dir ); if ( !-f "$Dir/LOCK" ) { @@ -152,162 +157,208 @@ my $tarCreateErrCnt = 1; # assume not ok until we learn otherwise my $tarCreateErr; my($logMsg, %stat, $xfer); -# -# Now do the restore -# -local(*RH, *WH); - $stat{xferOK} = $stat{hostAbort} = undef; $stat{hostError} = $stat{lastOutputLine} = undef; +local(*RH, *WH); # -# Create a pipe to connect BackupPC_tarCreate to the transport program -# (smbclient, tar, etc). -# WH is the write handle for writing, provided to BackupPC_tarCreate -# and RH is the other end of the pipe for reading provided to the -# transport program. +# Run an optional pre-restore command # -pipe(RH, WH); +UserCommandRun("RestorePreUserCmd"); +$NeedPostCmd = 1; -# -# Run the transport program, which reads from RH and extracts the data. -# -my $xferArgs = { - host => $host, - hostIP => $hostIP, - type => "restore", - shareName => $RestoreReq{shareDest}, - pipeRH => *RH, - pipeWH => *WH, - XferLOG => $RestoreLOG, -}; if ( $Conf{XferMethod} eq "tar" ) { # # Use tar (eg: tar/ssh) as the transport program. # - $xfer = BackupPC::Xfer::Tar->new($bpc, $xferArgs); + $xfer = BackupPC::Xfer::Tar->new($bpc); +} elsif ( $Conf{XferMethod} eq "rsync" || $Conf{XferMethod} eq "rsyncd" ) { + # + # Use rsync as the transport program. + # + if ( !defined($xfer = BackupPC::Xfer::Rsync->new($bpc)) ) { + my $errStr = BackupPC::Xfer::Rsync->errStr; + print(LOG $bpc->timeStamp, "restore failed: $errStr\n"); + print("restore failed: $errStr\n"); + UserCommandRun("RestorePostUserCmd") if ( $NeedPostCmd ); + exit(1); + } } else { # # Default is to use smbclient (smb) as the transport program. # - $xfer = BackupPC::Xfer::Smb->new($bpc, $xferArgs); -} -if ( !defined($logMsg = $xfer->start()) ) { - print(LOG $bpc->timeStamp, $xfer->errStr, "\n"); - print($xfer->errStr, "\n"); - exit(1); + $xfer = BackupPC::Xfer::Smb->new($bpc); } -# -# The parent must close the read handle since the transport program -# is using it. -# -close(RH); +my $useTar = $xfer->useTar; -# -# fork a child for BackupPC_tarCreate. TAR is a file handle -# on which we (the parent) read the stderr from BackupPC_tarCreate. -# -my @tarPathOpts; -if ( defined($RestoreReq{pathHdrDest}) - && $RestoreReq{pathHdrDest} ne $RestoreReq{pathHdrSrc} ) { - @tarPathOpts = ("-r", $RestoreReq{pathHdrSrc}, - "-p", $RestoreReq{pathHdrDest} - ); -} -my @tarArgs = ( - "-h", $RestoreReq{hostSrc}, - "-n", $RestoreReq{num}, - "-s", $RestoreReq{shareSrc}, - "-t", - @tarPathOpts, - @{$RestoreReq{fileList}}, -); -my $logMsg = "Running: $BinDir/BackupPC_tarCreate " - . join(" ", @tarArgs) . "\n"; -$RestoreLOG->write(\$logMsg); -if ( !defined($tarPid = open(TAR, "-|")) ) { - print(LOG $bpc->timeStamp, "can't fork to run tar\n"); - print("can't fork to run tar\n"); - close(WH); - # FIX: need to cleanup xfer - exit(0); -} -if ( !$tarPid ) { +if ( $useTar ) { # - # This is the tarCreate child. Clone STDERR to STDOUT, - # STDOUT to WH, and then exec BackupPC_tarCreate. + # Create a pipe to connect BackupPC_tarCreate to the transport program + # (smbclient, tar, etc). + # WH is the write handle for writing, provided to BackupPC_tarCreate + # and RH is the other end of the pipe for reading provided to the + # transport program. # - setpgrp 0,0; - close(STDERR); - open(STDERR, ">&STDOUT"); - close(STDOUT); - open(STDOUT, ">&WH"); - exec("$BinDir/BackupPC_tarCreate", @tarArgs); - print(LOG $bpc->timeStamp, "can't exec $BinDir/BackupPC_tarCreate\n"); - # FIX: need to cleanup xfer - exit(0); + pipe(RH, WH); } + # -# The parent must close the write handle since BackupPC_tarCreate -# is using it. +# Run the transport program, which reads from RH and extracts the data. # -close(WH); +my @Backups = $bpc->BackupInfoRead($RestoreReq{hostSrc}); +my $xferArgs = { + host => $host, + hostIP => $hostIP, + type => "restore", + shareName => $RestoreReq{shareDest}, + pipeRH => *RH, + pipeWH => *WH, + XferLOG => $RestoreLOG, + XferMethod => $Conf{XferMethod}, + bkupSrcHost => $RestoreReq{hostSrc}, + bkupSrcShare => $RestoreReq{shareSrc}, + bkupSrcNum => $RestoreReq{num}, + backups => \@Backups, + pathHdrSrc => $RestoreReq{pathHdrSrc}, + pathHdrDest => $RestoreReq{pathHdrDest}, + fileList => $RestoreReq{fileList}, +}; -$xferPid = $xfer->xferPid; -print(LOG $bpc->timeStamp, $logMsg, " (tarPid=$tarPid, xferPid=$xferPid)\n"); -print("started restore, tarPid=$tarPid, xferPid=$xferPid\n"); +$xfer->args($xferArgs); -# -# Parse the output of the transfer program and BackupPC_tarCreate -# while they run. Since we are reading from two or more children -# we use a select. -# -my($FDread, $tarOut, $mesg); -vec($FDread, fileno(TAR), 1) = 1; -$xfer->setSelectMask(\$FDread); +if ( !defined($logMsg = $xfer->start()) ) { + print(LOG $bpc->timeStamp, "xfer start failed: ", $xfer->errStr, "\n"); + print($xfer->errStr, "\n"); + UserCommandRun("RestorePostUserCmd") if ( $NeedPostCmd ); + exit(1); +} -SCAN: while ( 1 ) { - my $ein = $FDread; - last if ( $FDread =~ /^\0*$/ ); - select(my $rout = $FDread, undef, $ein, undef); - if ( vec($rout, fileno(TAR), 1) ) { - if ( sysread(TAR, $mesg, 8192) <= 0 ) { - vec($FDread, fileno(TAR), 1) = 0; - if ( !close(TAR) ) { - $tarCreateErrCnt = 1; - $tarCreateErr = "BackupPC_tarCreate failed"; - } - } else { - $tarOut .= $mesg; - } +if ( $useTar ) { + # + # Now do the restore by running BackupPC_tarCreate + # + # The parent must close the read handle since the transport program + # is using it. + # + close(RH); + + # + # fork a child for BackupPC_tarCreate. TAR is a file handle + # on which we (the parent) read the stderr from BackupPC_tarCreate. + # + my @tarPathOpts; + if ( defined($RestoreReq{pathHdrDest}) + && $RestoreReq{pathHdrDest} ne $RestoreReq{pathHdrSrc} ) { + @tarPathOpts = ("-r", $RestoreReq{pathHdrSrc}, + "-p", $RestoreReq{pathHdrDest} + ); } - while ( $tarOut =~ /(.*?)[\n\r]+(.*)/s ) { - $_ = $1; - $tarOut = $2; - $RestoreLOG->write(\"tarCreate: $_\n"); - if ( /^Done: (\d+) files, (\d+) bytes, (\d+) dirs, (\d+) specials, (\d+) errors/ ) { - $tarCreateFileCnt = $1; - $tarCreateByteCnt = $2; - $tarCreateErrCnt = $5; - } + my @tarArgs = ( + "-h", $RestoreReq{hostSrc}, + "-n", $RestoreReq{num}, + "-s", $RestoreReq{shareSrc}, + "-t", + @tarPathOpts, + @{$RestoreReq{fileList}}, + ); + my $logMsg = "Running: $BinDir/BackupPC_tarCreate " + . join(" ", @tarArgs) . "\n"; + $RestoreLOG->write(\$logMsg); + if ( !defined($tarPid = open(TAR, "-|")) ) { + print(LOG $bpc->timeStamp, "can't fork to run tar\n"); + print("can't fork to run tar\n"); + close(WH); + # FIX: need to cleanup xfer + UserCommandRun("RestorePostUserCmd") if ( $NeedPostCmd ); + exit(0); } - last if ( !$xfer->readOutput(\$FDread, $rout) ); - while ( my $str = $xfer->logMsgGet ) { - print(LOG $bpc->timeStamp, "xfer: $str\n"); + if ( !$tarPid ) { + # + # This is the tarCreate child. Clone STDERR to STDOUT, + # STDOUT to WH, and then exec BackupPC_tarCreate. + # + setpgrp 0,0; + close(STDERR); + open(STDERR, ">&STDOUT"); + close(STDOUT); + open(STDOUT, ">&WH"); + exec("$BinDir/BackupPC_tarCreate", @tarArgs); + print(LOG $bpc->timeStamp, "can't exec $BinDir/BackupPC_tarCreate\n"); + # FIX: need to cleanup xfer + exit(0); } - if ( $xfer->getStats->{fileCnt} == 1 ) { - # - # Make sure it is still the machine we expect. We do this while - # the transfer is running to avoid a potential race condition if - # the ip address was reassigned by dhcp just before we started - # the transfer. - # - if ( my $errMsg = CorrectHostCheck($hostIP, $host) ) { - $stat{hostError} = $errMsg; - last SCAN; - } + # + # The parent must close the write handle since BackupPC_tarCreate + # is using it. + # + close(WH); + + $xferPid = $xfer->xferPid; + print(LOG $bpc->timeStamp, $logMsg, " (tarPid=$tarPid, xferPid=$xferPid)\n"); + print("started restore, tarPid=$tarPid, xferPid=$xferPid\n"); + + # + # Parse the output of the transfer program and BackupPC_tarCreate + # while they run. Since we are reading from two or more children + # we use a select. + # + my($FDread, $tarOut, $mesg); + vec($FDread, fileno(TAR), 1) = 1; + $xfer->setSelectMask(\$FDread); + + SCAN: while ( 1 ) { + my $ein = $FDread; + last if ( $FDread =~ /^\0*$/ ); + alarm($Conf{ClientTimeout}); + select(my $rout = $FDread, undef, $ein, undef); + if ( vec($rout, fileno(TAR), 1) ) { + if ( sysread(TAR, $mesg, 8192) <= 0 ) { + vec($FDread, fileno(TAR), 1) = 0; + if ( !close(TAR) ) { + $tarCreateErrCnt = 1; + $tarCreateErr = "BackupPC_tarCreate failed"; + } + } else { + $tarOut .= $mesg; + } + } + while ( $tarOut =~ /(.*?)[\n\r]+(.*)/s ) { + $_ = $1; + $tarOut = $2; + $RestoreLOG->write(\"tarCreate: $_\n"); + if ( /^Done: (\d+) files, (\d+) bytes, (\d+) dirs, (\d+) specials, (\d+) errors/ ) { + $tarCreateFileCnt = $1; + $tarCreateByteCnt = $2; + $tarCreateErrCnt = $5; + } + } + last if ( !$xfer->readOutput(\$FDread, $rout) ); + while ( my $str = $xfer->logMsgGet ) { + print(LOG $bpc->timeStamp, "xfer: $str\n"); + } + if ( $xfer->getStats->{fileCnt} == 1 ) { + # + # Make sure it is still the machine we expect. We do this while + # the transfer is running to avoid a potential race condition if + # the ip address was reassigned by dhcp just before we started + # the transfer. + # + if ( my $errMsg = CorrectHostCheck($hostIP, $host) ) { + $stat{hostError} = $errMsg; + last SCAN; + } + } } +} else { + # + # otherwise the xfer module does everything for us + # + print(LOG $bpc->timeStamp, "Starting restore (tarPid=-1, xferPid=-1)\n"); + print("started restore, tarPid=-1, xferPid=-1\n"); + ($tarCreateFileCnt, $tarCreateByteCnt, + $tarCreateErrCnt, $tarCreateErr) = $xfer->run(); } +alarm(0); # # Merge the xfer status (need to accumulate counts) @@ -326,22 +377,22 @@ foreach my $k ( (keys(%stat), keys(%$newStat)) ) { next; } } -$RestoreLOG->close(); + $stat{xferOK} = 0 if ( $stat{hostError} || $stat{hostAbort} || $tarCreateErr ); if ( !$stat{xferOK} ) { # # kill off the tranfer program, first nicely then forcefully # - kill(2, $xferPid); + kill(2, $xferPid) if ( $xferPid > 0 ); sleep(1); - kill(9, $xferPid); + kill(9, $xferPid) if ( $xferPid > 0 ); # # kill off the tar process, first nicely then forcefully # - kill(2, $tarPid); + kill(2, $tarPid) if ( $tarPid > 0 ); sleep(1); - kill(9, $tarPid); + kill(9, $tarPid) if ( $tarPid > 0 ); } my $lastNum = -1; @@ -359,6 +410,13 @@ for ( my $i = 0 ; $i < @Restores ; $i++ ) { $lastNum = $Restores[$i]{num} if ( $lastNum < $Restores[$i]{num} ); } $lastNum++; + +# +# Run an optional post-restore command +# +UserCommandRun("RestorePostUserCmd") if ( $NeedPostCmd ); + +$RestoreLOG->close(); rename("$Dir/RestoreLOG$fileExt", "$Dir/RestoreLOG.$lastNum$fileExt"); rename("$Dir/$reqFileName", "$Dir/RestoreInfo.$lastNum"); my $endTime = time(); @@ -429,3 +487,69 @@ sub CorrectHostCheck if ( $netBiosHost ne $host ); return; } + +sub catch_signal +{ + my $signame = shift; + + # + # Note: needs to be tested for each kind of XferMethod + # + print(LOG $bpc->timeStamp, "cleaning up after signal $signame\n"); + if ( $xferPid > 0 ) { + if ( kill(2, $xferPid) <= 0 ) { + sleep(1); + kill(9, $xferPid); + } + } + if ( $tarPid > 0 ) { + if ( kill(2, $tarPid) <= 0 ) { + sleep(1); + kill(9, $tarPid); + } + } + $stat{xferOK} = 0; + $stat{hostError} = "aborted by signal $signame"; +} + +# +# Run an optional pre- or post-dump command +# +sub UserCommandRun +{ + my($type) = @_; + + return if ( !defined($Conf{$type}) ); + my $vars = { + xfer => $xfer, + host => $host, + hostIP => $hostIP, + share => $RestoreReq{shareDest}, + XferMethod => $Conf{XferMethod}, + LOG => *LOG, + XferLOG => $RestoreLOG, + stat => \%stat, + xferOK => $stat{xferOK}, + type => $type, + bkupSrcHost => $RestoreReq{hostSrc}, + bkupSrcShare => $RestoreReq{shareSrc}, + bkupSrcNum => $RestoreReq{num}, + backups => \@Backups, + pathHdrSrc => $RestoreReq{pathHdrSrc}, + pathHdrDest => $RestoreReq{pathHdrDest}, + fileList => $RestoreReq{fileList}, + }; + my $cmd = $bpc->cmdVarSubstitute($Conf{$type}, $vars); + $RestoreLOG->write(\"Executing $type: @$cmd\n"); + # + # Run the user's command, dumping the stdout/stderr into the + # Xfer log file. Also supply the optional $vars and %Conf in + # case the command is really perl code instead of a shell + # command. + # + $bpc->cmdSystemOrEval($cmd, + sub { + $RestoreLOG->write(\$_[0]); + }, + $vars, \%Conf); +} diff --git a/cgi-bin/BackupPC_Admin b/cgi-bin/BackupPC_Admin index 63f03af..57511ef 100755 --- a/cgi-bin/BackupPC_Admin +++ b/cgi-bin/BackupPC_Admin @@ -360,6 +360,9 @@ sub Action_View $comment = "(Extracting only Errors)"; } elsif ( $host ne "" && $type eq "config" ) { $file = "$TopDir/pc/$host/config.pl"; + $file = "$TopDir/conf/$host.pl" + if ( $host ne "config" && -f "$TopDir/conf/$host.pl" + && !-f $file ); } elsif ( $type eq "docs" ) { $file = "$BinDir/../doc/BackupPC.html"; if ( open(LOG, $file) ) { @@ -402,7 +405,8 @@ sub Action_View while ( 1 ) { $_ = $fh->readLine(); if ( $_ eq "" ) { - print(eval ("qq{$Lang->{skipped__skipped_lines}}")); + print(eval ("qq{$Lang->{skipped__skipped_lines}}")) + if ( $skipped ); last; } if ( /smb: \\>/ @@ -425,7 +429,8 @@ sub Action_View $skipped++; next; } - print(eval("qq{$Lang->{skipped__skipped_lines}}")) if ( $skipped ); + print(eval("qq{$Lang->{skipped__skipped_lines}}")) + if ( $skipped ); $skipped = 0; print ${EscapeHTML($_)}; } @@ -750,17 +755,16 @@ EOF } my @otherDirs; foreach my $i ( $view->backupList($share, $dir) ) { - next if ( $i == $num ); my $path = $dir; my $shareURI = $share; $path =~ s/([^\w.\/-])/uc sprintf("%%%02x", ord($1))/eg; $shareURI =~ s/([^\w.\/-])/uc sprintf("%%%02x", ord($1))/eg; - push(@otherDirs, <$i -EOF + push(@otherDirs, "$i"); + } if ( @otherDirs ) { - my $otherDirs = join(", ", @otherDirs); + my $otherDirs = join(",\n", @otherDirs); $filledBackup .= eval("qq{$Lang->{Visit_this_directory_in_backup}}"); } @@ -944,7 +948,7 @@ EOF Trailer(); } elsif ( $In{type} == 4 ) { if ( !defined($Hosts->{$In{hostDest}}) ) { - ErrorExit(eval("qq{$Lang->{Host_doesn_t_exist}}")); + ErrorExit(eval("qq{$Lang->{Host__doesn_t_exist}}")); } if ( !CheckPermission($In{hostDest}) ) { ErrorExit(eval("qq{$Lang->{You_don_t_have_permission_to_restore_onto_host}}")); @@ -1232,9 +1236,7 @@ EOF $MBNew EOF - $Backups[$i]{compress} ||= "off"; - my $is_compress = $Lang->{off}; - if ($Backups[$i]{compress} ne "off") {$is_compress = $Lang->{on}; } + my $is_compress = $Backups[$i]{compress} || $Lang->{off}; if (! $ExistComp) { $ExistComp = " "; } if (! $MBExistComp) { $MBExistComp = " "; } $compStr .= <{"${name}Time"}); $info->{"${name}FileCntRm"} = $info->{"${name}FileCntRm"} + 0; return eval("qq{$Lang->{Pool_Stat}}"); - } ########################################################################### @@ -1761,8 +1762,10 @@ sub Header { link => "?action=queue", name => $Lang->{Current_queues} }, { link => "?action=view&type=docs", name => $Lang->{Documentation}, priv => 1}, + { link => "http://backuppc.sourceforge.net/faq", name => "FAQ", + priv => 1}, { link => "http://backuppc.sourceforge.net", name => "SourceForge", - priv => 1}, + priv => 1}, ); print $Cgi->header(); print < # # COPYRIGHT -# Copyright (C) 2001 Craig Barratt +# Copyright (C) 2001-2003 Craig Barratt # # See http://backuppc.sourceforge.net. # @@ -152,6 +152,14 @@ $Conf{MaxOldLogFiles} = 14; # $Conf{DfPath} = '/bin/df'; +# +# Command to run df. Several variables are substituted at run-time: +# +# $dfPath path to df ($Conf{DfPath}) +# $topDir top-level BackupPC data directory +# +$Conf{DfCmd} = '$dfPath $topDir'; + # # Maximum threshold for disk utilization on the __TOPDIR__ filesystem. # If the output from $Conf{DfPath} reports a percentage larger than @@ -492,10 +500,18 @@ $Conf{BlackoutWeekDays} = [1, 2, 3, 4, 5]; # # The valid values are: # -# - 'smb': use smbclient and the SMB protocol. Only choice for WinXX. +# - 'smb': backup and restore via smbclient and the SMB protocol. +# Best choice for WinXX. +# +# - 'rsync': backup and restore via rsync (via rsh or ssh). +# Best choice for linux/unix. Can also work on WinXX. +# +# - 'rsyncd': backup and restre via rsync daemon on the client. +# Best choice for linux/unix if you have rsyncd running on +# the client. Can also work on WinXX. # -# - 'tar': use tar, tar over ssh, rsh or nfs. Best choice for -# linux/unix. +# - 'tar': backup and restore via tar, tar over ssh, rsh or nfs. +# Good choice for linux/unix. # # A future version should support 'rsync' as a transport method for # more efficient backup of linux/unix machines (and perhaps WinXX??). @@ -628,14 +644,38 @@ $Conf{TarClientPath} = '/bin/tar'; $Conf{RsyncClientPath} = '/bin/rsync'; # -# Full command to run rsync on the client machine +# Full command to run rsync on the client machine. The following variables +# are substituted at run-time: +# +# $host host name being backed up +# $hostIP host's IP address +# $shareName share name to backup (ie: top-level directory path) +# $rsyncPath same as $Conf{RsyncClientPath} +# $sshPath same as $Conf{SshPath} +# $argList argument list, built from $Conf{RsyncArgs}, +# $shareName, $Conf{BackupFilesExclude} and +# $Conf{BackupFilesOnly} +# +# This setting only matters if $Conf{XferMethod} = 'rsync'. # -$Conf{RsyncClientCmd} = '$sshPath -q -l root $host $rsyncPath $argList'; +$Conf{RsyncClientCmd} = '$sshPath -l root $host $rsyncPath $argList'; # -# Full command to run rsync for restore on the client. +# Full command to run rsync for restore on the client. The following +# variables are substituted at run-time: # -## $Conf{RsyncClientRestoreCmd} = ''; +# $host host name being backed up +# $hostIP host's IP address +# $shareName share name to backup (ie: top-level directory path) +# $rsyncPath same as $Conf{RsyncClientPath} +# $sshPath same as $Conf{SshPath} +# $argList argument list, built from $Conf{RsyncArgs}, +# $shareName, $Conf{BackupFilesExclude} and +# $Conf{BackupFilesOnly} +# +# This setting only matters if $Conf{XferMethod} = 'rsync'. +# +$Conf{RsyncClientRestoreCmd} = '$sshPath -l root $host $rsyncPath $argList'; # # Share name to backup. For $Conf{XferMethod} = "rsync" this should @@ -651,9 +691,42 @@ $Conf{RsyncShareName} = '/'; $Conf{RsyncdClientPort} = 873; # -# Key arguments to rsync server. Do not edit these unless you -# have a very thorough understanding of how File::RsyncP works. -# Really, do not edit these. See $Conf{RsyncClientArgs} instead. +# Rsync daemon user name on client, for $Conf{XferMethod} = "rsyncd". +# The user name and password are stored on the client in whatever file +# the "secrets file" parameter in rsyncd.conf points to +# (eg: /etc/rsyncd.secrets). +# +$Conf{RsyncdUserName} = ''; + +# +# Rsync daemon user name on client, for $Conf{XferMethod} = "rsyncd". +# The user name and password are stored on the client in whatever file +# the "secrets file" parameter in rsyncd.conf points to +# (eg: /etc/rsyncd.secrets). +# +$Conf{RsyncdPasswd} = ''; + +# +# Whether authentication is mandatory when connecting to the client's +# rsyncd. By default this is on, ensuring that BackupPC will refuse to +# connect to an rsyncd on the client that is not password protected. +# Turn off at your own risk. +# +$Conf{RsyncdAuthRequired} = 1; + +# +# Arguments to rsync for backup. Do not edit the first set unless you +# have a thorough understanding of how File::RsyncP works. +# +# Examples of additional arguments that should work are --exclude/--include, +# eg: +# +# $Conf{RsyncArgs} = [ +# # original arguments here +# '-v', +# '--exclude', '/proc', +# '--exclude', '*.tmp', +# ]; # $Conf{RsyncArgs} = [ # @@ -665,32 +738,48 @@ $Conf{RsyncArgs} = [ '--group', '--devices', '--links', + '--times', '--block-size=2048', - '--relative', '--recursive', + # + # Add additional arguments here + # ]; # -# Additional Rsync arguments that are given to the remote (client) -# rsync. Unfortunately you need a pretty good understanding of -# File::RsyncP to know which arguments will work; not all will. -# Examples that should work are --exclude/--include, eg: +# Arguments to rsync for restore. Do not edit the first set unless you +# have a thorough understanding of how File::RsyncP works. # -# $Conf{RsyncClientArgs} = [ -# '--exclude', '*.tmp', -# ]; # -$Conf{RsyncClientArgs} = [ +$Conf{RsyncRestoreArgs} = [ + # + # Do not edit these! + # + "--numeric-ids", + "--perms", + "--owner", + "--group", + "--devices", + "--links", + "--times", + "--block-size=2048", + "--relative", + "--ignore-times", + "--recursive", + # + # Add additional arguments here + # ]; # # Amount of verbosity in Rsync Xfer log files. 0 means be quiet, -# 1 will give some general information, 2 will give one line per file, -# 3 will include skipped files, higher values give more output. -# 10 will include byte dumps of all data read/written, which will -# make the log files huge. +# 1 will give will give one line per file, 2 will also show skipped +# files on incrementals, higher values give more output. 10 will +# include byte dumps of all data read/written, which will make the +# log files huge. # -$Conf{RsyncLogLevel} = 2; +$Conf{RsyncLogLevel} = 1; + # # Full path for ssh. Security caution: normal users should not # allowed to write to this file or directory. @@ -706,6 +795,14 @@ $Conf{SshPath} = '/usr/bin/ssh'; # $Conf{NmbLookupPath} = '/usr/bin/nmblookup'; +# +# NmbLookup command. Several variables are substituted at run-time: +# +# $nmbLookupPath path to nmblookup ($Conf{NmbLookupPath}) +# $host host name +# +$Conf{NmbLookupCmd} = '$nmbLookupPath -A $host'; + # # For fixed IP address hosts, BackupPC_dump can also verify the netbios # name to ensure it matches the host name. An error is generated if @@ -728,9 +825,12 @@ $Conf{FixedIPNetBiosNameCheck} = 0; $Conf{PingPath} = '/bin/ping'; # -# Options for the ping command. +# Ping command. Several variables are substituted at run-time: +# +# $pingPath path to ping ($Conf{PingPath}) +# $host host name # -$Conf{PingArgs} = '-c 1 $host'; +$Conf{PingCmd} = '$pingPath -c 1 $host'; # # Compression level to use on files. 0 means no compression. Compression @@ -788,7 +888,7 @@ $Conf{PingMaxMsec} = 20; # Despite the name, this parameter sets the timeout for all transport # methods (tar, smb etc). # -$Conf{SmbClientTimeout} = 7200; +$Conf{ClientTimeout} = 7200; # # Maximum number of log files we keep around in each PC's directory @@ -803,6 +903,29 @@ $Conf{SmbClientTimeout} = 7200; # $Conf{MaxOldPerPCLogFiles} = 12; +# +# Optional commands to run before and after dumps and restores. +# Stdout from these commands will be written to the Xfer (or Restore) +# log file. One example of using these commands would be to +# shut down and restart a database server, or to dump a database +# to files for backup. Example: +# +# $Conf{DumpPreUserCmd} = '$sshPath -l root $host /usr/bin/dumpMysql'; +# +# Various variable substitutions are available; see BackupPC_dump +# or BackupPC_restore for the details. +# +$Conf{DumpPreUserCmd} = undef; +$Conf{DumpPostUserCmd} = undef; +$Conf{RestorePreUserCmd} = undef; +$Conf{RestorePostUserCmd} = undef; + +# +# Advanced option for asking BackupPC to load additional perl modules. +# Can be a list (array ref) of module names to load at startup. +# +$Conf{PerlModuleLoad} = undef; + ########################################################################### # Email reminders, status and messages # (can be overridden in the per-PC config.pl) diff --git a/configure.pl b/configure.pl index b7a427a..a544aaf 100755 --- a/configure.pl +++ b/configure.pl @@ -15,7 +15,7 @@ # Craig Barratt # # COPYRIGHT -# Copyright (C) 2001 Craig Barratt +# Copyright (C) 2001-2003 Craig Barratt # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -127,7 +127,7 @@ my %Programs = ( rsync => "RsyncClientPath", ping => "PingPath", df => "DfPath", - 'ssh2/ssh' => "SshPath", + 'ssh/ssh2' => "SshPath", sendmail => "SendmailPath", hostname => "HostnamePath", ); @@ -456,20 +456,48 @@ if ( -f $dest ) { } $Conf{EMailFromUserName} ||= $Conf{BackupPCUser}; $Conf{EMailAdminUserName} ||= $Conf{BackupPCUser}; + +# +# Update various config parameters +# + # # IncrFill should now be off # $Conf{IncrFill} = 0; + # # Figure out sensible arguments for the ping command # -if ( $^O eq "solaris" || $^O eq "sunos" ) { - $Conf{PingArgs} ||= '-s $host 56 1'; -} elsif ( ($^O eq "linux" || $^O eq "openbsd" || $^O eq "netbsd") - && !system("$Conf{PingClientPath} -c 1 -w 3 localhost") ) { - $Conf{PingArgs} ||= '-c 1 -w 3 $host'; -} else { - $Conf{PingArgs} ||= '-c 1 $host'; +if ( defined($Conf{PingArgs}) ) { + $Conf{PingCmd} = '$pingPath ' . $Conf{PingArgs}; +} elsif ( !defined($Conf{PingCmd}) ) { + if ( $^O eq "solaris" || $^O eq "sunos" ) { + $Conf{PingCmd} = '$pingPath -s $host 56 1'; + } elsif ( ($^O eq "linux" || $^O eq "openbsd" || $^O eq "netbsd") + && !system("$Conf{PingClientPath} -c 1 -w 3 localhost") ) { + $Conf{PingCmd} = '$pingPath -c 1 -w 3 $host'; + } else { + $Conf{PingCmd} = '$pingPath -c 1 $host'; + } + delete($Conf{PingArgs}); +} + +# +# Figure out sensible arguments for the df command +# +if ( !defined($Conf{DfCmd}) ) { + if ( $^O eq "solaris" || $^O eq "sunos" ) { + $Conf{DfCmd} = '$dfPath -k $topDir'; + } +} + +# +# $Conf{SmbClientTimeout} is now $Conf{ClientTimeout} +# +if ( defined($Conf{SmbClientTimeout}) ) { + $Conf{ClientTimeout} = $Conf{SmbClientTimeout}; + delete($Conf{SmbClientTimeout}); } my $confCopy = "$dest.pre-__VERSION__"; @@ -518,7 +546,7 @@ if ( $Conf{CgiDir} ne "" ) { print <{$var} = @conf if ( defined($var) ); push(@conf, { text => $out, - var => $var, + var => $var, }); $out = $_; } else { diff --git a/doc-src/BackupPC.pod b/doc-src/BackupPC.pod index a7dd0e2..f5eda7b 100644 --- a/doc-src/BackupPC.pod +++ b/doc-src/BackupPC.pod @@ -234,15 +234,12 @@ releases of BackupPC: =item * -Adding support for rsync as a transport method, in addition to -smb and tar. This will give big savings in network traffic for -linux/unix clients. I haven't decided whether to save the pool file -rsync checksums (that would double the number of files in the pool, but -eliminate most server disk reads), or recompute them every time. I expect -to use native rsync on the client side. On the server, rsync would -need to understand the compressed file format, the file name mangling -and the attribute files, so I will either have to add features to rsync -or emulate rsync on the server side in perl. +Adding hardlink support to rsync. + +=item * + +Adding block and file checksum caching to rsync. This will significantly +increase performance. =item * @@ -253,10 +250,48 @@ you could request that you get sent email if any files below /bin, =item * -Resuming incomplete completed full backups. Useful if a machine +Resuming incomplete full backups. Useful if a machine (eg: laptop) is disconnected from the network during a backup, -or if the user manually stops a backup. This would work by -excluding directories that were already complete. +or if the user manually stops a backup. This would be supported +initially for rsync. The partial dump would be kept, and be +browsable. When the next dump starts, an incremental against +the partial dump would be done to make sure it was up to date, +and then the rest of the full dump would be done. + +=item * + +Replacing smbclient with the perl module FileSys::SmbClient. This +gives much more direct control of the smb transfer, allowing +incrementals to depend on any attribute change (eg: exist, mtime, +file size, uid, gid), and better support for include and exclude. +Currently smbclient incrementals only depend upon mtime, so +deleted files or renamed files are not detected. FileSys::SmbClient +would also allow resuming of incomplete full backups in the +same manner as rsync will. + +=item * + +Support --listed-incremental or --incremental for tar, +so that incrementals will depend upon any attribute change (eg: exist, +mtime, file size, uid, gid), rather than just mtime. This will allow +tar to be to as capable as FileSys::SmbClient and rsync. + +=item * + +For rysnc (and smb when FileSys::SmbClient is supported, and tar when +--listed-incremental is supported) support multi-level incrementals. +In fact, since incrementals will now be more "accurate", you could +choose to never to full dumps (except the first time), or at a +minimum do them infrequently: each incremental would depend upon +the last, giving a continuous chain of differential dumps. + +=item * + +Add a backup browsing feature that shows backup history by file. +So rather than a single directory view, it would be a table showing +the files (down) and the backups (across). The internal hardlinks +encode which files are identical across backups. You could immediately +see which files changed on which backups. =item * @@ -268,7 +303,9 @@ benchmarks on a large pool suggests that the potential savings are around 15-20%, which isn't spectacular, and likely not worth the implementation effort. The program xdelta (v1) on SourceForge (see L) uses an rsync algorithm for -doing efficient binary file deltas. +doing efficient binary file deltas. Rather than using an external +program, File::RsyncP will eventually get the necessary delta +generataion code from rsync. =back @@ -371,8 +408,10 @@ If you are using rsync to backup linux/unix machines you should have version 2.5.5 on each client machine. See L. Use "rsync --version" to check your version. -For BackupPC to use Rsync you will also need to install the perl Rsync -module, which is available from L. +For BackupPC to use Rsync you will also need to install the perl +File::RsyncP module, which is available from +L. +Version 0.20 is required. =item * @@ -430,13 +469,34 @@ Download the latest version from L. =head2 Step 2: Installing the distribution -First off, to enable compression, you will need to install Compress::Zlib -from L. It is optional, but strongly recommended. +First off, there are three perl modules you should install. +These are all optional, but highly recommended: + +=over 4 + +=item Compress::Zlib + +To enable compression, you will need to install Compress::Zlib +from L. +You can run "perldoc Compress::Zlib" to see if this module is installed. + +=item Archive::Zip + To support restore via Zip archives you will need to install -Archive::Zip, also from L. You can run -"perldoc Compress::Zlib" to see if this module is installed. -Finally, you will need the Rsync module. To build and install these -packages you should run these commands: +Archive::Zip, also from L. +You can run "perldoc Archive::Zip" to see if this module is installed. + +=item File::RsyncP + +To use rsync and rsyncd with BackupPC you will need to install File::RsyncP. +You can run "perldoc File::RsyncP" to see if this module is installed. +File::RsyncP is available from L. +Version 0.20 is required. + +=back + +To build and install these packages, fetch the tar.gz file and +then run these commands: tar zxvf Archive-Zip-1.01.tar.gz cd Archive-Zip-1.01 @@ -445,6 +505,8 @@ packages you should run these commands: make test make install +The same sequence of commands can be used for each module. + Now let's move onto BackupPC itself. After fetching BackupPC-__VERSION__.tar.gz, run these commands as root: @@ -721,17 +783,97 @@ root), since it needs sufficient permissions to read all the backup files. Ssh is setup so that BackupPC on the server (an otherwise low privileged user) can ssh as root on the client, without being prompted for a password. There are two common versions of ssh: v1 and v2. Here -are some instructions for one way to setup ssh v2: +are some instructions for one way to setup ssh. (Check which version +of SSH you have by typing "ssh" or "man ssh".) + +=over 4 + +=item OpenSSH Instructions =over 4 =item Key generation -As root on the client machine, use ssh2-keygen to generate a +As root on the client machine, use ssh-keygen to generate a +public/private key pair, without a pass-phrase: + + ssh-keygen -t rsa -N '' + +This will save the public key in ~/.ssh/id_rsa.pub and the private +key in ~/.ssh/id_rsa. + +=item BackupPC setup + +Repeat the above steps for the BackupPC user (__BACKUPPCUSER__) on the server. +Make a copy of the public key to make it recognizable, eg: + + ssh-keygen -t rsa -N '' + cp ~/.ssh/id_rsa.pub ~/.ssh/BackupPC_id_rsa.pub + +See the ssh and sshd manual pages for extra configuration information. + +=item Key exchange + +To allow BackupPC to ssh to the client as root, you need to place +BackupPC's public key into root's authorized list on the client. +Append BackupPC's public key (BackupPC_id_rsa.pub) to root's +~/.ssh/authorized_keys2 file on the client: + + touch ~/.ssh/authorized_keys2 + cat BackupPC_id_rsa.pub >> ~/.ssh/authorized_keys2 + +You should edit ~/.ssh/authorized_keys2 and add further specifiers, +eg: from, to limit which hosts can login using this key. For example, +if your BackupPC host is called backuppc.my.com, there should be +one line in ~/.ssh/authorized_keys2 that looks like: + + from="backuppc.my.com" ssh-rsa [base64 key, eg: ABwBCEAIIALyoqa8....] + +=item Fix permissions + +You will probably need to make sure that all the files +in ~/.ssh have no group or other read/write permission: + + chmod -R go-rwx ~/.ssh + +You should do the same thing for the BackupPC user on the server. + +=item Testing + +As the BackupPC user on the server, verify that this command: + + ssh -l root clientHostName whoami + +prints + + root + +You might be prompted the first time to accept the client's host key and +you might be prompted for root's password on the client. Make sure that +this command runs cleanly with no prompts after the first time. You +might need to check /etc/hosts.equiv on the client. Look at the +man pages for more information. The "-v" option to ssh is a good way +to get detailed information about what fails. + +=back + +=item SSH2 Instructions + +=over 4 + +=item Key generation + +As root on the client machine, use ssh-keygen2 to generate a public/private key pair, without a pass-phrase: ssh-keygen2 -t rsa -P +or: + + ssh-keygen -t rsa -N '' + +(This command might just be called ssh-keygen on your machine.) + This will save the public key in /.ssh2/id_rsa_1024_a.pub and the private key in /.ssh2/id_rsa_1024_a. @@ -795,7 +937,9 @@ might need to check /etc/hosts.equiv on the client. Look at the man pages for more information. The "-v" option to ssh2 is a good way to get detailed information about what fails. -=item ssh version 1 instructions +=back + +=item SSH version 1 Instructions The concept is identical and the steps are similar, but the specific commands and file names are slightly different. diff --git a/lib/BackupPC/Lang/en.pm b/lib/BackupPC/Lang/en.pm index a606679..581db27 100644 --- a/lib/BackupPC/Lang/en.pm +++ b/lib/BackupPC/Lang/en.pm @@ -828,7 +828,7 @@ $Lang{fileHeader} = < Name Type Mode - Backup# + # Size Mod time @@ -875,7 +875,7 @@ $Lang{The_directory_is_empty} = < EOF -$Lang{on} = "on"; +#$Lang{on} = "on"; $Lang{off} = "off"; $Lang{full} = "full"; diff --git a/lib/BackupPC/Lang/fr.pm b/lib/BackupPC/Lang/fr.pm index 44bf79f..573a440 100644 --- a/lib/BackupPC/Lang/fr.pm +++ b/lib/BackupPC/Lang/fr.pm @@ -835,7 +835,7 @@ $Lang{fileHeader} = < Nom Type Mode - Sauvegarde n° + n° Taille Date de modification @@ -883,7 +883,7 @@ $Lang{The_directory_is_empty} = < EOF -$Lang{on} = "actif"; +#$Lang{on} = "actif"; $Lang{off} = "inactif"; $Lang{full} = "complet"; diff --git a/lib/BackupPC/Lib.pm b/lib/BackupPC/Lib.pm index 5946f1b..f0590bc 100644 --- a/lib/BackupPC/Lib.pm +++ b/lib/BackupPC/Lib.pm @@ -54,7 +54,7 @@ sub new my $class = shift; my($topDir, $installDir) = @_; - my $self = bless { + my $bpc = bless { TopDir => $topDir || '/data/BackupPC', BinDir => $installDir || '/usr/local/BackupPC', LibDir => $installDir || '/usr/local/BackupPC', @@ -71,49 +71,49 @@ sub new tarCreateErrs xferErrs )], }, $class; - $self->{BinDir} .= "/bin"; - $self->{LibDir} .= "/lib"; + $bpc->{BinDir} .= "/bin"; + $bpc->{LibDir} .= "/lib"; # # Clean up %ENV and setup other variables. # delete @ENV{qw(IFS CDPATH ENV BASH_ENV)}; - $self->{PoolDir} = "$self->{TopDir}/pool"; - $self->{CPoolDir} = "$self->{TopDir}/cpool"; - if ( defined(my $error = $self->ConfigRead()) ) { + $bpc->{PoolDir} = "$bpc->{TopDir}/pool"; + $bpc->{CPoolDir} = "$bpc->{TopDir}/cpool"; + if ( defined(my $error = $bpc->ConfigRead()) ) { print(STDERR $error, "\n"); return; } - return $self; + return $bpc; } sub TopDir { - my($self) = @_; - return $self->{TopDir}; + my($bpc) = @_; + return $bpc->{TopDir}; } sub BinDir { - my($self) = @_; - return $self->{BinDir}; + my($bpc) = @_; + return $bpc->{BinDir}; } sub Version { - my($self) = @_; - return $self->{Version}; + my($bpc) = @_; + return $bpc->{Version}; } sub Conf { - my($self) = @_; - return %{$self->{Conf}}; + my($bpc) = @_; + return %{$bpc->{Conf}}; } sub Lang { - my($self) = @_; - return $self->{Lang}; + my($bpc) = @_; + return $bpc->{Lang}; } sub adminJob @@ -128,7 +128,7 @@ sub trashJob sub timeStamp { - my($self, $t, $noPad) = @_; + my($bpc, $t, $noPad) = @_; my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($t || time); $year += 1900; @@ -144,7 +144,7 @@ sub timeStamp # sub timeStampISO { - my($self, $t, $noPad) = @_; + my($bpc, $t, $noPad) = @_; my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($t || time); $year += 1900; @@ -156,17 +156,17 @@ sub timeStampISO sub BackupInfoRead { - my($self, $host) = @_; + my($bpc, $host) = @_; local(*BK_INFO, *LOCK); my(@Backups); - flock(LOCK, LOCK_EX) if open(LOCK, "$self->{TopDir}/pc/$host/LOCK"); - if ( open(BK_INFO, "$self->{TopDir}/pc/$host/backups") ) { + flock(LOCK, LOCK_EX) if open(LOCK, "$bpc->{TopDir}/pc/$host/LOCK"); + if ( open(BK_INFO, "$bpc->{TopDir}/pc/$host/backups") ) { while ( ) { s/[\n\r]+//; next if ( !/^(\d+\t(incr|full)[\d\t]*$)/ ); $_ = $1; - @{$Backups[@Backups]}{@{$self->{BackupFields}}} = split(/\t/); + @{$Backups[@Backups]}{@{$bpc->{BackupFields}}} = split(/\t/); } close(BK_INFO); } @@ -176,20 +176,20 @@ sub BackupInfoRead sub BackupInfoWrite { - my($self, $host, @Backups) = @_; + my($bpc, $host, @Backups) = @_; local(*BK_INFO, *LOCK); my($i); - flock(LOCK, LOCK_EX) if open(LOCK, "$self->{TopDir}/pc/$host/LOCK"); - unlink("$self->{TopDir}/pc/$host/backups.old") - if ( -f "$self->{TopDir}/pc/$host/backups.old" ); - rename("$self->{TopDir}/pc/$host/backups", - "$self->{TopDir}/pc/$host/backups.old") - if ( -f "$self->{TopDir}/pc/$host/backups" ); - if ( open(BK_INFO, ">$self->{TopDir}/pc/$host/backups") ) { + flock(LOCK, LOCK_EX) if open(LOCK, "$bpc->{TopDir}/pc/$host/LOCK"); + unlink("$bpc->{TopDir}/pc/$host/backups.old") + if ( -f "$bpc->{TopDir}/pc/$host/backups.old" ); + rename("$bpc->{TopDir}/pc/$host/backups", + "$bpc->{TopDir}/pc/$host/backups.old") + if ( -f "$bpc->{TopDir}/pc/$host/backups" ); + if ( open(BK_INFO, ">$bpc->{TopDir}/pc/$host/backups") ) { for ( $i = 0 ; $i < @Backups ; $i++ ) { my %b = %{$Backups[$i]}; - printf(BK_INFO "%s\n", join("\t", @b{@{$self->{BackupFields}}})); + printf(BK_INFO "%s\n", join("\t", @b{@{$bpc->{BackupFields}}})); } close(BK_INFO); } @@ -198,17 +198,17 @@ sub BackupInfoWrite sub RestoreInfoRead { - my($self, $host) = @_; + my($bpc, $host) = @_; local(*RESTORE_INFO, *LOCK); my(@Restores); - flock(LOCK, LOCK_EX) if open(LOCK, "$self->{TopDir}/pc/$host/LOCK"); - if ( open(RESTORE_INFO, "$self->{TopDir}/pc/$host/restores") ) { + flock(LOCK, LOCK_EX) if open(LOCK, "$bpc->{TopDir}/pc/$host/LOCK"); + if ( open(RESTORE_INFO, "$bpc->{TopDir}/pc/$host/restores") ) { while ( ) { s/[\n\r]+//; next if ( !/^(\d+.*)/ ); $_ = $1; - @{$Restores[@Restores]}{@{$self->{RestoreFields}}} = split(/\t/); + @{$Restores[@Restores]}{@{$bpc->{RestoreFields}}} = split(/\t/); } close(RESTORE_INFO); } @@ -218,21 +218,21 @@ sub RestoreInfoRead sub RestoreInfoWrite { - my($self, $host, @Restores) = @_; + my($bpc, $host, @Restores) = @_; local(*RESTORE_INFO, *LOCK); my($i); - flock(LOCK, LOCK_EX) if open(LOCK, "$self->{TopDir}/pc/$host/LOCK"); - unlink("$self->{TopDir}/pc/$host/restores.old") - if ( -f "$self->{TopDir}/pc/$host/restores.old" ); - rename("$self->{TopDir}/pc/$host/restores", - "$self->{TopDir}/pc/$host/restores.old") - if ( -f "$self->{TopDir}/pc/$host/restores" ); - if ( open(RESTORE_INFO, ">$self->{TopDir}/pc/$host/restores") ) { + flock(LOCK, LOCK_EX) if open(LOCK, "$bpc->{TopDir}/pc/$host/LOCK"); + unlink("$bpc->{TopDir}/pc/$host/restores.old") + if ( -f "$bpc->{TopDir}/pc/$host/restores.old" ); + rename("$bpc->{TopDir}/pc/$host/restores", + "$bpc->{TopDir}/pc/$host/restores.old") + if ( -f "$bpc->{TopDir}/pc/$host/restores" ); + if ( open(RESTORE_INFO, ">$bpc->{TopDir}/pc/$host/restores") ) { for ( $i = 0 ; $i < @Restores ; $i++ ) { my %b = %{$Restores[$i]}; printf(RESTORE_INFO "%s\n", - join("\t", @b{@{$self->{RestoreFields}}})); + join("\t", @b{@{$bpc->{RestoreFields}}})); } close(RESTORE_INFO); } @@ -241,13 +241,15 @@ sub RestoreInfoWrite sub ConfigRead { - my($self, $host) = @_; + my($bpc, $host) = @_; my($ret, $mesg, $config, @configs); - $self->{Conf} = (); - push(@configs, "$self->{TopDir}/conf/config.pl"); - push(@configs, "$self->{TopDir}/pc/$host/config.pl") - if ( defined($host) && -f "$self->{TopDir}/pc/$host/config.pl" ); + $bpc->{Conf} = (); + push(@configs, "$bpc->{TopDir}/conf/config.pl"); + push(@configs, "$bpc->{TopDir}/conf/$host.pl") + if ( $host ne "config" && -f "$bpc->{TopDir}/conf/$host.pl" ); + push(@configs, "$bpc->{TopDir}/pc/$host/config.pl") + if ( defined($host) && -f "$bpc->{TopDir}/pc/$host/config.pl" ); foreach $config ( @configs ) { %Conf = (); if ( !defined($ret = do $config) && ($! || $@) ) { @@ -256,17 +258,28 @@ sub ConfigRead $mesg =~ s/[\n\r]+//; return $mesg; } - %{$self->{Conf}} = ( %{$self->{Conf} || {}}, %Conf ); + %{$bpc->{Conf}} = ( %{$bpc->{Conf} || {}}, %Conf ); } - return if ( !defined($self->{Conf}{Language}) ); - my $langFile = "$self->{LibDir}/BackupPC/Lang/$self->{Conf}{Language}.pm"; + return if ( !defined($bpc->{Conf}{Language}) ); + if ( defined($bpc->{Conf}{PerlModuleLoad}) ) { + # + # Load any user-specified perl modules. This is for + # optional user-defined extensions. + # + $bpc->{Conf}{PerlModuleLoad} = [$bpc->{Conf}{PerlModuleLoad}] + if ( ref($bpc->{Conf}{PerlModuleLoad}) ne "ARRAY" ); + foreach my $module ( @{$bpc->{Conf}{PerlModuleLoad}} ) { + eval("use $module;"); + } + } + my $langFile = "$bpc->{LibDir}/BackupPC/Lang/$bpc->{Conf}{Language}.pm"; if ( !defined($ret = do $langFile) && ($! || $@) ) { $mesg = "Couldn't open language file $langFile: $!" if ( $! ); $mesg = "Couldn't execute language file $langFile: $@" if ( $@ ); $mesg =~ s/[\n\r]+//; return $mesg; } - $self->{Lang} = \%Lang; + $bpc->{Lang} = \%Lang; return; } @@ -275,12 +288,12 @@ sub ConfigRead # sub ConfigMTime { - my($self) = @_; - return (stat("$self->{TopDir}/conf/config.pl"))[9]; + my($bpc) = @_; + return (stat("$bpc->{TopDir}/conf/config.pl"))[9]; } # -# Returns information from the host file in $self->{TopDir}/conf/hosts. +# Returns information from the host file in $bpc->{TopDir}/conf/hosts. # With no argument a ref to a hash of hosts is returned. Each # hash contains fields as specified in the hosts file. With an # argument a ref to a single hash is returned with information @@ -288,13 +301,13 @@ sub ConfigMTime # sub HostInfoRead { - my($self, $host) = @_; + my($bpc, $host) = @_; my(%hosts, @hdr, @fld); local(*HOST_INFO); - if ( !open(HOST_INFO, "$self->{TopDir}/conf/hosts") ) { - print(STDERR $self->timeStamp, - "Can't open $self->{TopDir}/conf/hosts\n"); + if ( !open(HOST_INFO, "$bpc->{TopDir}/conf/hosts") ) { + print(STDERR $bpc->timeStamp, + "Can't open $bpc->{TopDir}/conf/hosts\n"); return {}; } while ( ) { @@ -325,8 +338,8 @@ sub HostInfoRead # sub HostsMTime { - my($self) = @_; - return (stat("$self->{TopDir}/conf/hosts"))[9]; + my($bpc) = @_; + return (stat("$bpc->{TopDir}/conf/hosts"))[9]; } # @@ -339,7 +352,7 @@ sub HostsMTime # sub RmTreeQuiet { - my($self, $pwd, $roots) = @_; + my($bpc, $pwd, $roots) = @_; my(@files, $root); if ( defined($roots) && length($roots) ) { @@ -363,7 +376,7 @@ sub RmTreeQuiet @files = $d->read; $d->close; @files = grep $_!~/^\.{1,2}$/, @files; - $self->RmTreeQuiet("$pwd/$root", \@files); + $bpc->RmTreeQuiet("$pwd/$root", \@files); chdir($pwd); rmdir($root) || rmdir($root); } else { @@ -378,7 +391,7 @@ sub RmTreeQuiet # sub RmTreeDefer { - my($self, $trashDir, $file) = @_; + my($bpc, $trashDir, $file) = @_; my($i, $f); return if ( !-e $file ); @@ -395,7 +408,7 @@ sub RmTreeDefer my($f) = $2; my($cwd) = Cwd::fastcwd(); $cwd = $1 if ( $cwd =~ /(.*)/ ); - $self->RmTreeQuiet($d, $f); + $bpc->RmTreeQuiet($d, $f); chdir($cwd) if ( $cwd ); } } @@ -405,7 +418,7 @@ sub RmTreeDefer # sub RmTreeTrashEmpty { - my($self, $trashDir) = @_; + my($bpc, $trashDir) = @_; my(@files); my($cwd) = Cwd::fastcwd(); @@ -417,7 +430,7 @@ sub RmTreeTrashEmpty $d->close; @files = grep $_!~/^\.{1,2}$/, @files; return 0 if ( !@files ); - $self->RmTreeQuiet($trashDir, \@files); + $bpc->RmTreeQuiet($trashDir, \@files); chdir($cwd) if ( $cwd ); return 1; } @@ -428,14 +441,14 @@ sub RmTreeTrashEmpty # sub ServerConnect { - my($self, $host, $port, $justConnect) = @_; + my($bpc, $host, $port, $justConnect) = @_; local(*FH); - return if ( defined($self->{ServerFD}) ); + return if ( defined($bpc->{ServerFD}) ); # # First try the unix-domain socket # - my $sockFile = "$self->{TopDir}/log/BackupPC.sock"; + my $sockFile = "$bpc->{TopDir}/log/BackupPC.sock"; socket(*FH, PF_UNIX, SOCK_STREAM, 0) || return "unix socket: $!"; if ( !connect(*FH, sockaddr_un($sockFile)) ) { my $err = "unix connect: $!"; @@ -453,14 +466,14 @@ sub ServerConnect } } my($oldFH) = select(*FH); $| = 1; select($oldFH); - $self->{ServerFD} = *FH; + $bpc->{ServerFD} = *FH; return if ( $justConnect ); # # Read the seed that we need for our MD5 message digest. See # ServerMesg below. # - sysread($self->{ServerFD}, $self->{ServerSeed}, 1024); - $self->{ServerMesgCnt} = 0; + sysread($bpc->{ServerFD}, $bpc->{ServerSeed}, 1024); + $bpc->{ServerMesgCnt} = 0; return; } @@ -469,13 +482,13 @@ sub ServerConnect # sub ServerOK { - my($self) = @_; + my($bpc) = @_; - return 0 if ( !defined($self->{ServerFD}) ); - vec(my $FDread, fileno($self->{ServerFD}), 1) = 1; + return 0 if ( !defined($bpc->{ServerFD}) ); + vec(my $FDread, fileno($bpc->{ServerFD}), 1) = 1; my $ein = $FDread; return 0 if ( select(my $rout = $FDread, undef, $ein, 0.0) < 0 ); - return 1 if ( !vec($rout, fileno($self->{ServerFD}), 1) ); + return 1 if ( !vec($rout, fileno($bpc->{ServerFD}), 1) ); } # @@ -483,10 +496,10 @@ sub ServerOK # sub ServerDisconnect { - my($self) = @_; - return if ( !defined($self->{ServerFD}) ); - close($self->{ServerFD}); - delete($self->{ServerFD}); + my($bpc) = @_; + return if ( !defined($bpc->{ServerFD}) ); + close($bpc->{ServerFD}); + delete($bpc->{ServerFD}); } # @@ -507,13 +520,13 @@ sub ServerDisconnect # sub ServerMesg { - my($self, $mesg) = @_; - return if ( !defined(my $fh = $self->{ServerFD}) ); + my($bpc, $mesg) = @_; + return if ( !defined(my $fh = $bpc->{ServerFD}) ); my $md5 = Digest::MD5->new; - $md5->add($self->{ServerSeed} . $self->{ServerMesgCnt} - . $self->{Conf}{ServerMesgSecret} . $mesg); + $md5->add($bpc->{ServerSeed} . $bpc->{ServerMesgCnt} + . $bpc->{Conf}{ServerMesgSecret} . $mesg); print($fh $md5->b64digest . " $mesg\n"); - $self->{ServerMesgCnt}++; + $bpc->{ServerMesgCnt}++; return <$fh>; } @@ -522,12 +535,12 @@ sub ServerMesg # sub ChildInit { - my($self) = @_; + my($bpc) = @_; close(STDERR); open(STDERR, ">&STDOUT"); select(STDERR); $| = 1; select(STDOUT); $| = 1; - $ENV{PATH} = $self->{Conf}{MyPath}; + $ENV{PATH} = $bpc->{Conf}{MyPath}; } # @@ -545,7 +558,7 @@ sub ChildInit # sub File2MD5 { - my($self, $md5, $name) = @_; + my($bpc, $md5, $name) = @_; my($data, $fileSize); local(*N); @@ -590,7 +603,7 @@ sub File2MD5 # sub Buffer2MD5 { - my($self, $md5, $fileSize, $dataRef) = @_; + my($bpc, $md5, $fileSize, $dataRef) = @_; $md5->reset(); $md5->add($fileSize); @@ -617,16 +630,16 @@ sub Buffer2MD5 # sub MD52Path { - my($self, $d, $compress, $poolDir) = @_; + my($bpc, $d, $compress, $poolDir) = @_; return if ( $d !~ m{(.)(.)(.)(.*)} ); - $poolDir = ($compress ? $self->{CPoolDir} : $self->{PoolDir}) + $poolDir = ($compress ? $bpc->{CPoolDir} : $bpc->{PoolDir}) if ( !defined($poolDir) ); return "$poolDir/$1/$2/$3/$1$2$3$4"; } # -# For each file, check if the file exists in $self->{TopDir}/pool. +# For each file, check if the file exists in $bpc->{TopDir}/pool. # If so, remove the file and make a hardlink to the file in # the pool. Otherwise, if the newFile flag is set, make a # hardlink in the pool to the new file. @@ -639,12 +652,12 @@ sub MD52Path # sub MakeFileLink { - my($self, $name, $d, $newFile, $compress) = @_; + my($bpc, $name, $d, $newFile, $compress) = @_; my($i, $rawFile); return -1 if ( !-f $name ); for ( $i = -1 ; ; $i++ ) { - return -2 if ( !defined($rawFile = $self->MD52Path($d, $compress)) ); + return -2 if ( !defined($rawFile = $bpc->MD52Path($d, $compress)) ); $rawFile .= "_$i" if ( $i >= 0 ); if ( -f $rawFile ) { if ( !compare($name, $rawFile) ) { @@ -666,64 +679,60 @@ sub MakeFileLink sub CheckHostAlive { - my($self, $host) = @_; - my($s, $pingArgs); + my($bpc, $host) = @_; + my($s, $pingCmd); - $pingArgs = $self->{Conf}{PingArgs}; - # - # Merge variables into $pingArgs - # - my $vars = { - host => $host, + my $args = { + pingPath => $bpc->{Conf}{PingPath}, + host => $host, }; - $pingArgs =~ s/\$(\w+)/defined($vars->{$1}) - ? $self->shellEscape($vars->{$1}) - : \$$1/eg; + $pingCmd = $bpc->cmdVarSubstitute($bpc->{Conf}{PingCmd}, $args); + # # Do a first ping in case the PC needs to wakeup # - $s = `$self->{Conf}{PingPath} $pingArgs 2>&1`; + $s = $bpc->cmdSystemOrEval($pingCmd, undef, $args); return -1 if ( $? ); + # # Do a second ping and get the round-trip time in msec # - $s = `$self->{Conf}{PingPath} $pingArgs 2>&1`; + $s = $bpc->cmdSystemOrEval($pingCmd, undef, $args); return -1 if ( $? ); - return $1 if ( $s !~ /time=([\d\.]+)\s*ms/ ); - return $1/1000 if ( $s !~ /time=([\d\.]+)\s*usec/ ); + return $1 if ( $s =~ /time=([\d\.]+)\s*ms/i ); + return $1/1000 if ( $s =~ /time=([\d\.]+)\s*usec/i ); return 0; } sub CheckFileSystemUsage { - my($self) = @_; - my($topDir) = $self->{TopDir}; - my($s); - - if ( $^O eq "solaris" ) { - $s = `$self->{Conf}{DfPath} -k $topDir 2>&1`; - return 0 if ( $? || $s !~ /(\d+)%/s ); - return $1; - } elsif ( $^O eq "sunos" ) { - $s = `$self->{Conf}{DfPath} $topDir 2>&1`; - return 0 if ( $? || $s !~ /(\d+)%/s ); - return $1; - } elsif ( $^O eq "linux" ) { - $s = `$self->{Conf}{DfPath} $topDir 2>&1`; - return 0 if ( $? || $s !~ /(\d+)%/s ); - return $1; - } else { - return 0; - } + my($bpc) = @_; + my($topDir) = $bpc->{TopDir}; + my($s, $dfCmd); + + my $args = { + dfPath => $bpc->{Conf}{DfPath}, + topDir => $bpc->{TopDir}, + }; + $dfCmd = $bpc->cmdVarSubstitute($bpc->{Conf}{DfCmd}, $args); + $s = $bpc->cmdSystemOrEval($dfCmd, undef, $args); + return 0 if ( $? || $s !~ /(\d+)%/s ); + return $1; } sub NetBiosInfoGet { - my($self, $host) = @_; + my($bpc, $host) = @_; my($netBiosHostName, $netBiosUserName); + my($s, $nmbCmd); - foreach ( split(/[\n\r]+/, `$self->{Conf}{NmbLookupPath} -A $host 2>&1`) ) { - next if ( !/([\w-]+)\s*<(\w{2})\> - .*/i ); + my $args = { + nmbLookupPath => $bpc->{Conf}{NmbLookupPath}, + host => $host, + }; + $nmbCmd = $bpc->cmdVarSubstitute($bpc->{Conf}{NmbLookupCmd}, $args); + foreach ( split(/[\n\r]+/, $bpc->cmdSystemOrEval($nmbCmd, undef, $args)) ) { + next if ( !/^\s*([\w\s-]+?)\s*<(\w{2})\> - .*/i ); $netBiosHostName ||= $1 if ( $2 eq "00" ); # host is first 00 $netBiosUserName = $1 if ( $2 eq "03" ); # user is last 03 } @@ -733,7 +742,7 @@ sub NetBiosInfoGet sub fileNameEltMangle { - my($self, $name) = @_; + my($bpc, $name) = @_; return "" if ( $name eq "" ); $name =~ s{([%/\n\r])}{sprintf("%%%02x", ord($1))}eg; @@ -749,10 +758,10 @@ sub fileNameEltMangle # sub fileNameMangle { - my($self, $name) = @_; + my($bpc, $name) = @_; - $name =~ s{/([^/]+)}{"/" . $self->fileNameEltMangle($1)}eg; - $name =~ s{^([^/]+)}{$self->fileNameEltMangle($1)}eg; + $name =~ s{/([^/]+)}{"/" . $bpc->fileNameEltMangle($1)}eg; + $name =~ s{^([^/]+)}{$bpc->fileNameEltMangle($1)}eg; return $name; } @@ -761,7 +770,7 @@ sub fileNameMangle # sub fileNameUnmangle { - my($self, $name) = @_; + my($bpc, $name) = @_; $name =~ s{/f}{/}g; $name =~ s{^f}{}; @@ -776,10 +785,134 @@ sub fileNameUnmangle # sub shellEscape { - my($self, $cmd) = @_; + my($bpc, $cmd) = @_; $cmd =~ s/([][;&()<>{}|^\n\r\t *\$\\'"`?])/\\$1/g; return $cmd; } +# +# Do variable substitution prior to execution of a command. +# +sub cmdVarSubstitute +{ + my($bpc, $template, $vars) = @_; + my(@cmd); + + # + # Return without any substitution if the first entry starts with "&", + # indicating this is perl code. + # + if ( (ref($template) eq "ARRAY" ? $template->[0] : $template) =~ /^\&/ ) { + return $template; + } + $template = [split(/\s+/, $template)] if ( ref($template) ne "ARRAY" ); + # + # Merge variables into @tarClientCmd + # + foreach my $arg ( @$template ) { + # + # Replace scalar variables first + # + $arg =~ s{\$(\w+)(\+?)}{ + defined($vars->{$1}) && ref($vars->{$1}) ne "ARRAY" + ? ($2 eq "+" ? $bpc->shellEscape($vars->{$1}) : $vars->{$1}) + : "\$$1" + }eg; + # + # Now replicate any array arguments; this just works for just one + # array var in each argument. + # + if ( $arg =~ m{(.*)\$(\w+)(\+?)(.*)} && ref($vars->{$2}) eq "ARRAY" ) { + my $pre = $1; + my $var = $2; + my $esc = $3; + my $post = $4; + foreach my $v ( @{$vars->{$var}} ) { + $v = $bpc->shellEscape($v) if ( $esc eq "+" ); + push(@cmd, "$pre$v$post"); + } + } else { + push(@cmd, $arg); + } + } + return \@cmd; +} + +# +# Exec or eval a command. $cmd is either a string on an array ref. +# +# @args are optional arguments for the eval() case; they are not used +# for exec(). +# +sub cmdExecOrEval +{ + my($bpc, $cmd, @args) = @_; + + if ( (ref($cmd) eq "ARRAY" ? $cmd->[0] : $cmd) =~ /^\&/ ) { + $cmd = join(" ", $cmd) if ( ref($cmd) eq "ARRAY" ); + eval($cmd) + } else { + $cmd = [split(/\s+/, $cmd)] if ( ref($cmd) ne "ARRAY" ); + exec(@$cmd); + } +} + +# +# System or eval a command. $cmd is either a string on an array ref. +# $stdoutCB is a callback for output generated by the command. If it +# is undef then output is returned. If it is a code ref then the function +# is called with each piece of output as an argument. If it is a scalar +# ref the output is appended to this variable. +# +# @args are optional arguments for the eval() case; they are not used +# for system(). +# +# Also, $? should be set when the CHILD pipe is closed. +# +sub cmdSystemOrEval +{ + my($bpc, $cmd, $stdoutCB, @args) = @_; + my($pid, $out); + local(*CHILD); + + if ( (ref($cmd) eq "ARRAY" ? $cmd->[0] : $cmd) =~ /^\&/ ) { + $cmd = join(" ", $cmd) if ( ref($cmd) eq "ARRAY" ); + my $out = eval($cmd); + $$stdoutCB .= $out if ( ref($stdoutCB) eq 'SCALAR' ); + &$stdoutCB($out) if ( ref($stdoutCB) eq 'CODE' ); + return $out if ( !defined($stdoutCB) ); + return; + } else { + $cmd = [split(/\s+/, $cmd)] if ( ref($cmd) ne "ARRAY" ); + if ( !defined($pid = open(CHILD, "-|")) ) { + my $err = "Can't fork to run @$cmd\n"; + $? = 1; + $$stdoutCB .= $err if ( ref($stdoutCB) eq 'SCALAR' ); + &$stdoutCB($err) if ( ref($stdoutCB) eq 'CODE' ); + return $err if ( !defined($stdoutCB) ); + return; + } + if ( !$pid ) { + # + # This is the child + # + close(STDERR); + open(STDERR, ">&STDOUT"); + exec(@$cmd); + } + # + # The parent gathers the output from the child + # + while ( ) { + $$stdoutCB .= $_ if ( ref($stdoutCB) eq 'SCALAR' ); + &$stdoutCB($_) if ( ref($stdoutCB) eq 'CODE' ); + $out .= $_ if ( !defined($stdoutCB) ); + } + $? = 0; + close(CHILD); + } + return $out; +} + 1; diff --git a/lib/BackupPC/View.pm b/lib/BackupPC/View.pm index 4cb6da2..be87096 100644 --- a/lib/BackupPC/View.pm +++ b/lib/BackupPC/View.pm @@ -302,24 +302,39 @@ sub find { my($m, $backupNum, $share, $path, $depth, $callback, @callbackArgs) = @_; + # + # First call the callback on the given $path + # + my $attr = $m->fileAttrib($backupNum, $share, $path); + return -1 if ( !defined($attr) ); + &$callback($attr, @callbackArgs); + return if ( $attr->{type} != BPC_FTYPE_DIR ); + + # + # Now recurse into subdirectories + # + $m->findRecurse($backupNum, $share, $path, $depth, + $callback, @callbackArgs); +} + +# +# Same as find(), except the callback is not called on the current +# $path, only on the contents of $path. So if $path is a file then +# no callback or recursion occurs. +# +sub findRecurse +{ + my($m, $backupNum, $share, $path, $depth, $callback, @callbackArgs) = @_; + my $attr = $m->dirAttrib($backupNum, $share, $path); - if ( !defined($attr) ) { - # - # maybe this is a file, not a directory; if so call the callback - # just on this file. - # - my $attr = $m->fileAttrib($backupNum, $share, $path); - return -1 if ( !defined($attr) ); - &$callback($attr, @callbackArgs); - return; - } + return if ( !defined($attr) ); foreach my $file ( keys(%$attr) ) { &$callback($attr->{$file}, @callbackArgs); next if ( !$depth || $attr->{$file}{type} != BPC_FTYPE_DIR ); # # For depth-first, recurse as we hit each directory # - $m->find($backupNum, $share, "$path/$file", $depth, + $m->findRecurse($backupNum, $share, "$path/$file", $depth, $callback, @callbackArgs); } if ( !$depth ) { @@ -328,7 +343,7 @@ sub find # foreach my $file ( keys(%{$attr}) ) { next if ( $attr->{$file}{type} != BPC_FTYPE_DIR ); - $m->find($backupNum, $share, "$path/$file", $depth, + $m->findRecurse($backupNum, $share, "$path/$file", $depth, $callback, @callbackArgs); } } diff --git a/lib/BackupPC/Xfer/Rsync.pm b/lib/BackupPC/Xfer/Rsync.pm index a6d61a1..b07c33b 100644 --- a/lib/BackupPC/Xfer/Rsync.pm +++ b/lib/BackupPC/Xfer/Rsync.pm @@ -41,7 +41,7 @@ use strict; use BackupPC::View; use BackupPC::Xfer::RsyncFileIO; -use vars qw( $RsyncLibOK ); +use vars qw( $RsyncLibOK $RsyncLibErr ); BEGIN { eval "use File::RsyncP;"; @@ -50,8 +50,14 @@ BEGIN { # Rsync module doesn't exist. # $RsyncLibOK = 0; + $RsyncLibErr = "File::RsyncP module doesn't exist"; } else { - $RsyncLibOK = 1; + if ( $File::RsyncP::VERSION < 0.20 ) { + $RsyncLibOK = 0; + $RsyncLibErr = "File::RsyncP module version too old: need 0.20"; + } else { + $RsyncLibOK = 1; + } } }; @@ -68,6 +74,20 @@ sub new hostIP => "", shareName => "", badFiles => [], + + # + # Various stats + # + byteCnt => 0, + fileCnt => 0, + xferErrCnt => 0, + xferBadShareCnt => 0, + xferBadFileCnt => 0, + xferOK => 0, + + # + # User's args + # %$args, }, $class; @@ -93,21 +113,32 @@ sub start my($t) = @_; my $bpc = $t->{bpc}; my $conf = $t->{conf}; - my(@fileList, @rsyncClientCmd, $logMsg, $incrDate); + my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg, + $incrDate, $argList, $fioArgs); + + # + # We add a slash to the share name we pass to rsync + # + ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{}; if ( $t->{type} eq "restore" ) { - # TODO - #push(@rsyncClientCmd, split(/ +/, $c o n f->{RsyncClientRestoreCmd})); - $logMsg = "restore not supported for $t->{shareName}"; - # - # restores are considered to work unless we see they fail - # (opposite to backups...) - # - $t->{xferOK} = 1; + $rsyncClientCmd = $conf->{RsyncClientRestoreCmd}; + $rsyncArgs = $conf->{RsyncRestoreArgs}; + my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}"; + $remoteDir =~ s{//+}{/}g; + $argList = ['--server', @$rsyncArgs, '.', $remoteDir]; + $fioArgs = { + host => $t->{bkupSrcHost}, + share => $t->{bkupSrcShare}, + viewNum => $t->{bkupSrcNum}, + fileList => $t->{fileList}, + }; + $logMsg = "restore started below directory $t->{shareName}" + . " to host $t->{host}"; } else { # # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude} - # into a hash of arrays of files. NOT IMPLEMENTED YET. + # into a hash of arrays of files. # $conf->{RsyncShareName} = [ $conf->{RsyncShareName} ] unless ref($conf->{RsyncShareName}) eq "ARRAY"; @@ -125,20 +156,62 @@ sub start }; } } + if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) { + my(@inc, @exc, %incDone, %excDone); + foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) { + # + # If the user wants to just include /home/craig, then + # we need to do create include/exclude pairs at + # each level: + # --include /home --exclude /* + # --include /home/craig --exclude /home/* + # + # It's more complex if the user wants to include multiple + # deep paths. For example, if they want /home/craig and + # /var/log, then we need this mouthfull: + # --include /home --include /var --exclude /* + # --include /home/craig --exclude /home/* + # --include /var/log --exclude /var/* + # + # To make this easier we do all the includes first and all + # of the excludes at the end (hopefully they commute). + # + $file = "/$file"; + $file =~ s{//+}{/}g; + my $f = ""; + while ( $file =~ m{^/([^/]*)(.*)} ) { + my $elt = $1; + $file = $2; + if ( $file eq "/" ) { + # + # preserve a tailing slash + # + $file = ""; + $elt = "$elt/"; + } + push(@exc, "$f/*") if ( !$excDone{"$f/*"} ); + $excDone{"$f/*"} = 1; + $f = "$f/$elt"; + push(@inc, $f) if ( !$incDone{$f} ); + $incDone{$f} = 1; + } + } + foreach my $file ( @inc ) { + push(@fileList, "--include=$file"); + } + foreach my $file ( @exc ) { + push(@fileList, "--exclude=$file"); + } + } if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) { foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} ) { + # + # just append additional exclude lists onto the end + # push(@fileList, "--exclude=$file"); } } - if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) { - foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) { - push(@fileList, $file); - } - } else { - push(@fileList, "."); - } - push(@rsyncClientCmd, split(/ +/, $conf->{RsyncClientCmd})); if ( $t->{type} eq "full" ) { $logMsg = "full backup started for directory $t->{shareName}"; } else { @@ -146,80 +219,75 @@ sub start $logMsg = "incr backup started back to $incrDate for directory" . " $t->{shareName}"; } - $t->{xferOK} = 0; - } - # - # Merge variables into @rsyncClientCmd - # - my $vars = { - host => $t->{host}, - hostIP => $t->{hostIP}, - shareName => $t->{shareName}, - rsyncPath => $conf->{RsyncClientPath}, - sshPath => $conf->{SshPath}, - }; - my @cmd = @rsyncClientCmd; - @rsyncClientCmd = (); - foreach my $arg ( @cmd ) { - next if ( $arg =~ /^\s*$/ ); - if ( $arg =~ /^\$fileList(\+?)/ ) { - my $esc = $1 eq "+"; - foreach $arg ( @fileList ) { - $arg = $bpc->shellEscape($arg) if ( $esc ); - push(@rsyncClientCmd, $arg); - } - } elsif ( $arg =~ /^\$argList(\+?)/ ) { - my $esc = $1 eq "+"; - foreach $arg ( (@{$conf->{RsyncArgs}}, - @{$conf->{RsyncClientArgs}}) ) { - $arg = $bpc->shellEscape($arg) if ( $esc ); - push(@rsyncClientCmd, $arg); - } - } else { - $arg =~ s{\$(\w+)(\+?)}{ - defined($vars->{$1}) - ? ($2 eq "+" ? $bpc->shellEscape($vars->{$1}) : $vars->{$1}) - : "\$$1" - }eg; - push(@rsyncClientCmd, $arg); - } + + # + # A full dump is implemented with --ignore-times: this causes all + # files to be checksummed, even if the attributes are the same. + # That way all the file contents are checked, but you get all + # the efficiencies of rsync: only files deltas need to be + # transferred, even though it is a full dump. + # + $rsyncArgs = $conf->{RsyncArgs}; + $rsyncArgs = [@$rsyncArgs, "--ignore-times"] + if ( $t->{type} eq "full" ); + $rsyncClientCmd = $conf->{RsyncClientCmd}; + $argList = ['--server', '--sender', @$rsyncArgs, + '.', $t->{shareNameSlash}]; + $fioArgs = { + host => $t->{host}, + share => $t->{shareName}, + viewNum => $t->{lastFullBkupNum}, + }; } # - # A full dump is implemented with --ignore-times: this causes all - # files to be checksummed, even if the attributes are the same. - # That way all the file contents are checked, but you get all - # the efficiencies of rsync: only files deltas need to be - # transferred, even though it is a full dump. + # Merge variables into $rsyncClientCmd # - my $rsyncArgs = $conf->{RsyncArgs}; - $rsyncArgs = [@$rsyncArgs, "--ignore-times"] if ( $t->{type} eq "full" ); + $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, + { + host => $t->{host}, + hostIP => $t->{hostIP}, + shareName => $t->{shareName}, + shareNameSlash => $t->{shareNameSlash}, + rsyncPath => $conf->{RsyncClientPath}, + sshPath => $conf->{SshPath}, + argList => $argList, + }); # # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO # module, which handles all the special BackupPC file storage # (compression, mangling, hardlinks, special files, attributes etc). # + $t->{rsyncClientCmd} = $rsyncClientCmd; $t->{rs} = File::RsyncP->new({ - logLevel => $conf->{RsyncLogLevel}, - rsyncCmd => \@rsyncClientCmd, - rsyncArgs => $rsyncArgs, - logHandler => sub { + logLevel => $conf->{RsyncLogLevel}, + rsyncCmd => sub { + $bpc->cmdExecOrEval($rsyncClientCmd); + }, + rsyncCmdType => "full", + rsyncArgs => $rsyncArgs, + logHandler => sub { my($str) = @_; $str .= "\n"; $t->{XferLOG}->write(\$str); - }, - fio => BackupPC::Xfer::RsyncFileIO->new({ + }, + fio => BackupPC::Xfer::RsyncFileIO->new({ xfer => $t, bpc => $t->{bpc}, conf => $t->{conf}, - host => $t->{host}, backups => $t->{backups}, logLevel => $conf->{RsyncLogLevel}, + timeout => $conf->{ClientTimeout}, + logHandler => sub { + my($str) = @_; + $str .= "\n"; + $t->{XferLOG}->write(\$str); + }, + %$fioArgs, }), }); - # TODO: alarm($conf->{SmbClientTimeout}); delete($t->{_errStr}); return $logMsg; @@ -230,12 +298,27 @@ sub run my($t) = @_; my $rs = $t->{rs}; my $conf = $t->{conf}; + my($remoteSend, $remoteDir, $remoteDirDaemon); + alarm($conf->{ClientTimeout}); + if ( $t->{type} eq "restore" ) { + $remoteSend = 0; + ($remoteDir = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g; + ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g; + $remoteDirDaemon = $t->{shareNameSlash} + if ( $t->{pathHdrDest} eq "" + || $t->{pathHdrDest} eq "/" ); + } else { + $remoteSend = 1; + $remoteDir = $t->{shareNameSlash}; + $remoteDirDaemon = "."; + } if ( $t->{XferMethod} eq "rsync" ) { # # Run rsync command # - $rs->remoteStart(1, $t->{shareName}); + $t->{XferLOG}->write(\"Running: @{$t->{rsyncClientCmd}}\n"); + $rs->remoteStart($remoteSend, $remoteDir); } else { # # Connect to the rsync server @@ -245,14 +328,22 @@ sub run $t->{hostError} = $err; return; } - if ( defined(my $err = $rs->serverService($t->{shareName}, - "craig", "xyz123", 0)) ) { + # + # Pass module name, and follow it with a slash if it already + # contains a slash; otherwise just keep the plain module name. + # + my $module = $t->{shareName}; + $module = $t->{shareNameSlash} if ( $module =~ /\// ); + if ( defined(my $err = $rs->serverService($module, + $conf->{RsyncdUserName}, + $conf->{RsyncdPasswd}, + $conf->{RsyncdAuthRequired})) ) { $t->{hostError} = $err; return; } - $rs->serverStart(1, "."); + $rs->serverStart($remoteSend, $remoteDirDaemon); } - my $error = $rs->go($t->{shareName}); + my $error = $rs->go($t->{shareNameSlash}); $rs->serverClose(); # @@ -261,9 +352,6 @@ sub run # $rs->{stats}{totalWritten} # $rs->{stats}{totalSize} # - # qw(byteCnt fileCnt xferErrCnt xferBadShareCnt xferBadFileCnt - # xferOK hostAbort hostError lastOutputLine) - # my $stats = $rs->statsFinal; if ( !defined($error) && defined($stats) ) { $t->{xferOK} = 1; @@ -279,23 +367,30 @@ sub run # $t->{hostError} = $error if ( defined($error) ); - return ( - 0, - $stats->{childStats}{ExistFileCnt} - + $stats->{parentStats}{ExistFileCnt}, - $stats->{childStats}{ExistFileSize} - + $stats->{parentStats}{ExistFileSize}, - $stats->{childStats}{ExistFileCompSize} - + $stats->{parentStats}{ExistFileCompSize}, - $stats->{childStats}{TotalFileCnt} - + $stats->{parentStats}{TotalFileCnt}, - $stats->{childStats}{TotalFileSize} - + $stats->{parentStats}{TotalFileSize}, - ); + if ( $t->{type} eq "restore" ) { + return ( + $t->{fileCnt}, + $t->{byteCnt}, + 0, + 0 + ); + } else { + return ( + 0, + $stats->{childStats}{ExistFileCnt} + + $stats->{parentStats}{ExistFileCnt}, + $stats->{childStats}{ExistFileSize} + + $stats->{parentStats}{ExistFileSize}, + $stats->{childStats}{ExistFileCompSize} + + $stats->{parentStats}{ExistFileCompSize}, + $stats->{childStats}{TotalFileCnt} + + $stats->{parentStats}{TotalFileCnt}, + $stats->{childStats}{TotalFileSize} + + $stats->{parentStats}{TotalFileSize}, + ); + } } -# alarm($conf->{SmbClientTimeout}); - sub setSelectMask { my($t, $FDreadRef) = @_; @@ -305,6 +400,7 @@ sub errStr { my($t) = @_; + return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" ); return $t->{_errStr}; } diff --git a/lib/BackupPC/Xfer/RsyncFileIO.pm b/lib/BackupPC/Xfer/RsyncFileIO.pm index d7c19fb..87c50bc 100644 --- a/lib/BackupPC/Xfer/RsyncFileIO.pm +++ b/lib/BackupPC/Xfer/RsyncFileIO.pm @@ -23,7 +23,8 @@ package BackupPC::Xfer::RsyncFileIO; use strict; use File::Path; use BackupPC::Attrib qw(:all); -use BackupPC::FileZIO; +use BackupPC::View; +use BackupPC::PoolWrite; use BackupPC::PoolWrite; use Data::Dumper; @@ -62,17 +63,24 @@ sub new digest => File::RsyncP::Digest->new, checksumSeed => 0, attrib => {}, + logHandler => \&logHandler, + stats => { + TotalFileCnt => 0, + TotalFileSize => 0, + ExistFileCnt => 0, + ExistFileSize => 0, + ExistFileCompSize => 0, + }, %$options, }, $class; - $fio->{shareM} = $fio->{bpc}->fileNameEltMangle($fio->{xfer}{shareName}); + $fio->{shareM} = $fio->{bpc}->fileNameEltMangle($fio->{share}); $fio->{outDir} = "$fio->{xfer}{outDir}/new/"; $fio->{outDirSh} = "$fio->{outDir}/$fio->{shareM}/"; $fio->{view} = BackupPC::View->new($fio->{bpc}, $fio->{host}, $fio->{backups}); - $fio->{full} = $fio->{xfer}{type} eq "full" ? 1 : 0; + $fio->{full} = $fio->{xfer}{type} eq "full" ? 1 : 0; $fio->{newFilesFH} = $fio->{xfer}{newFilesFH}; - $fio->{lastBkupNum} = $fio->{xfer}{lastBkupNum}; return $fio; } @@ -84,23 +92,36 @@ sub blockSize return $fio->{blockSize}; } +sub logHandlerSet +{ + my($fio, $sub) = @_; + $fio->{logHandler} = $sub; +} + # # Setup rsync checksum computation for the given file. # sub csumStart { - my($fio, $f) = @_; - my $attr = $fio->attribGet($f); + my($fio, $f, $needMD4) = @_; + my $attr = $fio->attribGet($f); $fio->{file} = $f; $fio->csumEnd if ( defined($fio->{fh}) ); return if ( $attr->{type} != BPC_FTYPE_FILE ); if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath}, 0, $attr->{compress})) ) { - $fio->log("Can't open $attr->{fullPath}"); + $fio->log("Can't open $attr->{fullPath} (name=$f->{name})"); return -1; } + if ( $needMD4) { + $fio->{csumDigest} = File::RsyncP::Digest->new; + $fio->{csumDigest}->add(pack("V", $fio->{checksumSeed})); + } else { + delete($fio->{csumDigest}); + } + alarm($fio->{timeout}) if ( defined($fio->{timeout}) ); } sub csumGet @@ -113,15 +134,15 @@ sub csumGet return if ( !defined($fio->{fh}) ); if ( $fio->{fh}->read(\$fileData, $blockSize * $num) <= 0 ) { - return $fio->csumEnd; + return; } - #$fileData = substr($fileData, 0, $blockSize * $num - 2); + $fio->{csumDigest}->add($fileData) if ( defined($fio->{csumDigest}) ); $fio->log(sprintf("%s: getting csum ($num,$csumLen,%d,0x%x)\n", $fio->{file}{name}, length($fileData), $fio->{checksumSeed})) if ( $fio->{logLevel} >= 10 ); - return $fio->{digest}->rsyncChecksum($fileData, $blockSize, + return $fio->{digest}->blockDigest($fileData, $blockSize, $csumLen, $fio->{checksumSeed}); } @@ -130,35 +151,49 @@ sub csumEnd my($fio) = @_; return if ( !defined($fio->{fh}) ); + # + # make sure we read the entire file for the file MD4 digest + # + if ( defined($fio->{csumDigest}) ) { + my $fileData; + while ( $fio->{fh}->read(\$fileData, 65536) > 0 ) { + $fio->{csumDigest}->add($fileData); + } + } $fio->{fh}->close(); delete($fio->{fh}); + return $fio->{csumDigest}->digest if ( defined($fio->{csumDigest}) ); } sub readStart { my($fio, $f) = @_; - my $attr = $fio->attribGet($f); + my $attr = $fio->attribGet($f); $fio->{file} = $f; $fio->readEnd if ( defined($fio->{fh}) ); - if ( !defined(my $fh = BackupPC::FileZIO->open($attr->{fullPath}, + if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath}, 0, $attr->{compress})) ) { - $fio->log("Can't open $attr->{fullPath}"); + $fio->log("Can't open $attr->{fullPath} (name=$f->{name})"); return; } + $fio->log("$f->{name}: opened for read") if ( $fio->{logLevel} >= 4 ); + alarm($fio->{timeout}) if ( defined($fio->{timeout}) ); } sub read { my($fio, $num) = @_; - my($fileData); + my $fileData; $num ||= 32768; return if ( !defined($fio->{fh}) ); if ( $fio->{fh}->read(\$fileData, $num) <= 0 ) { return $fio->readEnd; } + $fio->log(sprintf("read returns %d bytes", length($fileData))) + if ( $fio->{logLevel} >= 8 ); return \$fileData; } @@ -168,7 +203,9 @@ sub readEnd return if ( !defined($fio->{fh}) ); $fio->{fh}->close; + $fio->log("closing $fio->{file}{name})") if ( $fio->{logLevel} >= 8 ); delete($fio->{fh}); + return; } sub checksumSeed @@ -193,7 +230,7 @@ sub viewCacheDir #$fio->log("viewCacheDir($share, $dir)"); if ( !defined($share) ) { - $share = $fio->{xfer}{shareName}; + $share = $fio->{share}; $shareM = $fio->{shareM}; } else { $shareM = $fio->{bpc}->fileNameEltMangle($share); @@ -211,7 +248,7 @@ sub viewCacheDir # fetch new directory attributes # $fio->{viewCache}{$shareM} - = $fio->{view}->dirAttrib($fio->{lastBkupNum}, $share, $dir); + = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir); } sub attribGet @@ -219,19 +256,22 @@ sub attribGet my($fio, $f) = @_; my($dir, $fname, $share, $shareM); - if ( $f->{name} =~ m{(.*)/(.*)} ) { + $fname = $f->{name}; + $fname = "$fio->{xfer}{pathHdrSrc}/$fname" + if ( defined($fio->{xfer}{pathHdrSrc}) ); + $fname =~ s{//+}{/}g; + if ( $fname =~ m{(.*)/(.*)} ) { $shareM = $fio->{shareM}; $dir = $1; $fname = $2; - } elsif ( $f->{name} ne "." ) { + } elsif ( $fname ne "." ) { $shareM = $fio->{shareM}; $dir = ""; - $fname = $f->{name}; } else { $share = ""; $shareM = ""; $dir = ""; - $fname = $fio->{xfer}{shareName}; + $fname = $fio->{share}; } $fio->viewCacheDir($share, $dir); $shareM .= "/$dir" if ( $dir ne "" ); @@ -274,7 +314,7 @@ sub attribSet $dir = "$fio->{shareM}/" . $1; } elsif ( $f->{name} eq "." ) { $dir = ""; - $file = $fio->{xfer}{shareName}; + $file = $fio->{share}; } else { $dir = $fio->{shareM}; $file = $f->{name}; @@ -434,7 +474,7 @@ sub statsGet # # Make a given directory. Returns non-zero on error. # -sub mkpath +sub makePath { my($fio, $f) = @_; my $name = $1 if ( $f->{name} =~ /(.*)/ ); @@ -446,7 +486,7 @@ sub mkpath $path = $fio->{outDirSh} . $fio->{bpc}->fileNameMangle($name); } $fio->logFileAction("create", $f) if ( $fio->{logLevel} >= 1 ); - $fio->log("mkpath($path, 0777)") if ( $fio->{logLevel} >= 5 ); + $fio->log("makePath($path, 0777)") if ( $fio->{logLevel} >= 5 ); $path = $1 if ( $path =~ /(.*)/ ); File::Path::mkpath($path, 0, 0777) if ( !-d $path ); return $fio->attribSet($f) if ( -d $path ); @@ -457,7 +497,7 @@ sub mkpath # # Make a special file. Returns non-zero on error. # -sub mkspecial +sub makeSpecial { my($fio, $f) = @_; my $name = $1 if ( $f->{name} =~ /(.*)/ ); @@ -467,7 +507,7 @@ sub mkspecial my $str = ""; my $type = $fio->mode2type($f->{mode}); - $fio->log("mkspecial($path, $type, $f->{mode})") + $fio->log("makeSpecial($path, $type, $f->{mode})") if ( $fio->{logLevel} >= 5 ); if ( $type == BPC_FTYPE_CHARDEV || $type == BPC_FTYPE_BLOCKDEV ) { my($major, $minor, $fh, $fileData); @@ -488,6 +528,7 @@ sub mkspecial || $attr->{type} != $fio->mode2type($f->{mode}) || $attr->{mtime} != $f->{mtime} || $attr->{size} != $f->{size} + || $attr->{uid} != $f->{uid} || $attr->{gid} != $f->{gid} || $attr->{mode} != $f->{mode} || !defined($fh = BackupPC::FileZIO->open($attr->{fullPath}, 0, @@ -517,14 +558,26 @@ sub unlink } # -# Appends to list of log messages +# Default log handler +# +sub logHandler +{ + my($str) = @_; + + print(STDERR $str, "\n"); +} + +# +# Handle one or more log messages # sub log { - my($fio, @msg) = @_; + my($fio, @logStr) = @_; - $fio->{log} ||= []; - push(@{$fio->{log}}, @msg); + foreach my $str ( @logStr ) { + next if ( $str eq "" ); + $fio->{logHandler}($str); + } } # @@ -547,15 +600,13 @@ sub logFileAction } # -# Returns a list of log messages +# Later we'll use this function to complete a prior unfinished dump. +# We'll do an incremental on the part we have already, and then a +# full or incremental against the rest. # -sub logMsg +sub ignoreAttrOnFile { - my($fio) = @_; - my $log = $fio->{log} || []; - - delete($fio->{log}); - return @$log; + return undef; } # @@ -583,6 +634,7 @@ sub fileDeltaRxStart delete($fio->{rxOutFd}); delete($fio->{rxDigest}); delete($fio->{rxInData}); + alarm($fio->{timeout}) if ( defined($fio->{timeout}) ); } # @@ -781,14 +833,14 @@ sub fileDeltaRxDone } $fh->close; } else { - # error + # ERROR } $fio->log("$name got exact match") if ( $fio->{logLevel} >= 5 ); } close($fio->{rxInFd}) if ( defined($fio->{rxInFd}) ); unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" ); - my $newDigest = $fio->{rxDigest}->rsyncDigest; + my $newDigest = $fio->{rxDigest}->digest; if ( $fio->{logLevel} >= 3 ) { my $md4Str = unpack("H*", $md4); my $newStr = unpack("H*", $newDigest); @@ -864,33 +916,93 @@ sub fileDeltaRxDone return; } +# +# Callback function for BackupPC::View->find. Note the order of the +# first two arguments. +# sub fileListEltSend { - my($fio, $name, $fList, $outputFunc) = @_; - my @s = stat($name); - - (my $n = $name) =~ s/^\Q$fio->{localDir}/$fio->{remoteDir}/; - $fList->encode({ - fname => $n, - dev => $s[0], - inode => $s[1], - mode => $s[2], - uid => $s[4], - gid => $s[5], - rdev => $s[6], - mtime => $s[9], - }); + my($a, $fio, $fList, $outputFunc) = @_; + my $name = $a->{relPath}; + my $n = $name; + my $type = $fio->mode2type($a->{mode}); + my $extraAttribs = {}; + + $n =~ s/^\Q$fio->{xfer}{pathHdrSrc}//; + $fio->log("Sending $name (remote=$n)") if ( $fio->{logLevel} >= 4 ); + if ( $type == BPC_FTYPE_CHARDEV + || $type == BPC_FTYPE_BLOCKDEV + || $type == BPC_FTYPE_SYMLINK ) { + my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0, $a->{compress}); + my $str; + if ( defined($fh) ) { + if ( $fh->read(\$str, $a->{size} + 1) == $a->{size} ) { + if ( $type == BPC_FTYPE_SYMLINK ) { + # + # Reconstruct symbolic link + # + $extraAttribs = { link => $str }; + } elsif ( $str =~ /(\d*),(\d*)/ ) { + # + # Reconstruct char or block special major/minor device num + # + $extraAttribs = { rdev => $1 * 256 + $2 }; + } else { + # ERROR + $fio->log("$name: unexpected file contents $str"); + } + } else { + # ERROR + $fio->log("$name: can't read exactly $a->{size} bytes"); + } + $fh->close; + } else { + # ERROR + $fio->log("$name: can't open"); + } + } + my $f = { + name => $n, + #dev => 0, # later, when we support hardlinks + #inode => 0, # later, when we support hardlinks + mode => $a->{mode}, + uid => $a->{uid}, + gid => $a->{gid}, + mtime => $a->{mtime}, + size => $a->{size}, + %$extraAttribs, + }; + $fList->encode($f); + $f->{name} = "$fio->{xfer}{pathHdrDest}/$f->{name}"; + $f->{name} =~ s{//+}{/}g; + $fio->logFileAction("restore", $f) if ( $fio->{logLevel} >= 1 ); &$outputFunc($fList->encodeData); + # + # Cumulate stats + # + if ( $type != BPC_FTYPE_DIR ) { + $fio->{stats}{TotalFileCnt}++; + $fio->{stats}{TotalFileSize} += $a->{size}; + } + alarm($fio->{timeout}) if ( defined($fio->{timeout}) ); } sub fileListSend { my($fio, $flist, $outputFunc) = @_; - $fio->log("fileListSend not implemented!!"); - $fio->{view}->find($fio->{lastBkupNum}, $fio->{xfer}{shareName}, - $fio->{restoreFiles}, 1, \&fileListEltSend, - $flist, $outputFunc); + # + # Populate the file list with the files requested by the user. + # Since some might be directories so we call BackupPC::View::find. + # + $fio->log("fileListSend: sending file list: " + . join(" ", @{$fio->{fileList}})) if ( $fio->{logLevel} >= 4 ); + foreach my $name ( @{$fio->{fileList}} ) { + $fio->{view}->find($fio->{xfer}{bkupSrcNum}, + $fio->{xfer}{bkupSrcShare}, + $name, 1, + \&fileListEltSend, $fio, $flist, $outputFunc); + } } sub finish @@ -900,16 +1012,16 @@ sub finish # # Flush the attributes if this is the child # - $fio->attribWrite(undef) + $fio->attribWrite(undef); + alarm($fio->{timeout}) if ( defined($fio->{timeout}) ); } - -sub is_tainted -{ - return ! eval { - join('',@_), kill 0; - 1; - }; -} +#sub is_tainted +#{ +# return ! eval { +# join('',@_), kill 0; +# 1; +# }; +#} 1; diff --git a/lib/BackupPC/Xfer/Smb.pm b/lib/BackupPC/Xfer/Smb.pm index 06c1e4d..124b6f0 100644 --- a/lib/BackupPC/Xfer/Smb.pm +++ b/lib/BackupPC/Xfer/Smb.pm @@ -202,7 +202,7 @@ sub start return; } $t->{XferLOG}->write(\"Running: $smbClientCmd\n"); - alarm($conf->{SmbClientTimeout}); + alarm($conf->{ClientTimeout}); $t->{_errStr} = undef; return $logMsg; } @@ -228,7 +228,7 @@ sub readOutput # # refresh our inactivity alarm # - alarm($conf->{SmbClientTimeout}); + alarm($conf->{ClientTimeout}); $t->{lastOutputLine} = $_ if ( !/^$/ ); # # This section is highly dependent on the version of smbclient. diff --git a/lib/BackupPC/Xfer/Tar.pm b/lib/BackupPC/Xfer/Tar.pm index 7092b89..60326e1 100644 --- a/lib/BackupPC/Xfer/Tar.pm +++ b/lib/BackupPC/Xfer/Tar.pm @@ -78,11 +78,15 @@ sub start my($t) = @_; my $bpc = $t->{bpc}; my $conf = $t->{conf}; - my(@fileList, @tarClientCmd, $logMsg, $incrDate); + my(@fileList, $tarClientCmd, $logMsg, $incrDate); local(*TAR); if ( $t->{type} eq "restore" ) { - push(@tarClientCmd, split(/ +/, $conf->{TarClientRestoreCmd})); + if ( ref($conf->{TarClientRestoreCmd}) eq "ARRAY" ) { + $tarClientCmd = $conf->{TarClientRestoreCmd}; + } else { + $tarClientCmd = [split(/ +/, $conf->{TarClientRestoreCmd})]; + } $logMsg = "restore started below directory $t->{shareName}"; # # restores are considered to work unless we see they fail @@ -125,52 +129,35 @@ sub start } else { push(@fileList, "."); } + if ( ref($conf->{TarClientCmd}) eq "ARRAY" ) { + $tarClientCmd = $conf->{TarClientCmd}; + } else { + $tarClientCmd = [split(/ +/, $conf->{TarClientCmd})]; + } + my $args; if ( $t->{type} eq "full" ) { - push(@tarClientCmd, - split(/ +/, $conf->{TarClientCmd}), - split(/ +/, $conf->{TarFullArgs}) - ); + $args = $conf->{TarFullArgs}; $logMsg = "full backup started for directory $t->{shareName}"; } else { $incrDate = $bpc->timeStampISO($t->{lastFull} - 3600, 1); - push(@tarClientCmd, - split(/ +/, $conf->{TarClientCmd}), - split(/ +/, $conf->{TarIncrArgs}) - ); + $args = $conf->{TarIncrArgs}; $logMsg = "incr backup started back to $incrDate for directory" . " $t->{shareName}"; } + push(@$tarClientCmd, split(/ +/, $args)); } # # Merge variables into @tarClientCmd # - my $vars = { + $tarClientCmd = $bpc->cmdVarSubstitute($tarClientCmd, { host => $t->{host}, hostIP => $t->{hostIP}, incrDate => $incrDate, shareName => $t->{shareName}, + fileList => \@fileList, tarPath => $conf->{TarClientPath}, sshPath => $conf->{SshPath}, - }; - my @cmd = @tarClientCmd; - @tarClientCmd = (); - foreach my $arg ( @cmd ) { - next if ( $arg =~ /^\s*$/ ); - if ( $arg =~ /^\$fileList(\+?)/ ) { - my $esc = $1 eq "+"; - foreach $arg ( @fileList ) { - $arg = $bpc->shellEscape($arg) if ( $esc ); - push(@tarClientCmd, $arg); - } - } else { - $arg =~ s{\$(\w+)(\+?)}{ - defined($vars->{$1}) - ? ($2 eq "+" ? $bpc->shellEscape($vars->{$1}) : $vars->{$1}) - : "\$$1" - }eg; - push(@tarClientCmd, $arg); - } - } + }); if ( !defined($t->{xferPid} = open(TAR, "-|")) ) { $t->{_errStr} = "Can't fork to run tar"; return; @@ -204,13 +191,13 @@ sub start # # Run the tar command # - exec(@tarClientCmd); + $bpc->cmdExecOrEval($tarClientCmd); # should not be reached, but just in case... - $t->{_errStr} = "Can't exec @tarClientCmd"; + $t->{_errStr} = "Can't exec @$tarClientCmd"; return; } - $t->{XferLOG}->write(\"Running: @tarClientCmd\n"); - alarm($conf->{SmbClientTimeout}); + $t->{XferLOG}->write(\"Running: @$tarClientCmd\n"); + alarm($conf->{ClientTimeout}); $t->{_errStr} = undef; return $logMsg; } @@ -239,7 +226,7 @@ sub readOutput # # refresh our inactivity alarm # - alarm($conf->{SmbClientTimeout}); + alarm($conf->{ClientTimeout}); $t->{lastOutputLine} = $_ if ( !/^$/ ); if ( /^Total bytes written: / ) { $t->{xferOK} = 1; diff --git a/makeDist b/makeDist index 0d236f7..c6ebf3d 100755 --- a/makeDist +++ b/makeDist @@ -1,6 +1,38 @@ #!/bin/perl # -# Build a BackupPC distribution +# makeDist: Build a BackupPC distribution +# +# DESCRIPTION +# +# This script should be run with no arguments to build a +# distribution. The $Version and $ReleaseDate should be +# edited below to specify the version name and the release +# date. The distribution is createede in the sub-directory +# dist. The dsitribution is in the file name: +# +# dist/BackupPC-$Version.tar.gz. +# +# AUTHOR +# Craig Barratt +# +# COPYRIGHT +# Copyright (C) 2001-2003 Craig Barratt +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +#======================================================================== # use strict; @@ -9,8 +41,8 @@ use File::Copy; umask(0022); -my $Version = "1.6.0_CVS"; -my $ReleaseDate = "10 Dec 2002"; +my $Version = "2.0.0_CVS"; +my $ReleaseDate = "18 Jan 2003"; my $DistDir = "dist/BackupPC-$Version"; my @PerlSrc = qw( @@ -56,6 +88,7 @@ $ConfVars->{CgiImageDir} = 2; foreach my $file ( @PerlSrc ) { $errCnt += CheckConfigParams($file, $ConfVars, 1); } +$errCnt += CheckLangUsage(); exit(1) if ( $errCnt ); foreach my $var ( sort(keys(%$ConfVars) ) ) { @@ -107,6 +140,8 @@ foreach my $file ( (@PerlSrc, rmtree("doc", 0, 0); system("cd dist ; tar zcf BackupPC-$Version.tar.gz BackupPC-$Version"); print("Distribution written to dist/BackupPC-$Version.tar.gz\n"); +unlink("pod2htmd.x~~"); +unlink("pod2htmi.x~~"); ########################################################################### # Subroutines @@ -223,13 +258,19 @@ sub CheckConfigParams open(F, $file) || die("can't open $file\n"); if ( $check ) { while ( ) { - s/\$self->{Conf}{([^}\$]+)}/if ( !defined($vars->{$1}) ) { + s/\$(self|bpc)->{Conf}{([^}\$]+)}/if ( !defined($vars->{$2}) ) { + print("Unexpected Conf var $2 in $file\n"); + $errors++; + } else { + $vars->{$2}++; + }/eg; + s/\$[Cc]onf(?:->)?{([^}\$]+)}/if ( !defined($vars->{$1}) ) { print("Unexpected Conf var $1 in $file\n"); $errors++; } else { $vars->{$1}++; }/eg; - s/\$[Cc]onf(?:->)?{([^}\$]+)}/if ( !defined($vars->{$1}) ) { + s/UserCommandRun\("([^"]*)"\)/if ( !defined($vars->{$1}) ) { print("Unexpected Conf var $1 in $file\n"); $errors++; } else { @@ -245,3 +286,49 @@ sub CheckConfigParams close(F); return $errors; } + +# +# Make sure that every lang variable in cgi-bin/BackupPC_Admin matches +# the strings in each lib/BackupPC/Lang/*.pm file. This makes sure +# we didn't miss any translations in any of the languages. +# +sub CheckLangUsage +{ + my $errors; + my $vars = {}; + + open(F, "cgi-bin/BackupPC_Admin") + || die("can't open cgi-bin/BackupPC_Admin\n"); + while ( ) { + s/\$Lang->{([^}]*)}/$vars->{$1} = 1;/eg; + } + close(F); + foreach my $f ( ) { + my $done = {}; + open(F, $f) || die("can't open $f\n"); + while ( ) { + s/#.*//g; + s/\$Lang{([^}]*)}/ + my $var = $1; + next if ( $var =~ m{^(Reason_|Status_)} ); + if ( !defined($vars->{$var}) ) { + print("Unexpected Lang var $var in $f\n"); + $errors++; + } else { + $done->{$var} = 1; + }/eg; + } + close(F); + foreach my $v ( keys(%$vars) ) { + # + # skip "variables" with "$", since they are like expressions + # + next if ( $v =~ /\$/ ); + if ( !defined($done->{$v}) ) { + print("Lang var $v missing from $f\n"); + $errors++; + } + } + } + return $errors; +} -- 2.20.1