* Commit for 2.1.0.
[BackupPC.git] / lib / BackupPC / Xfer / Rsync.pm
index a6d61a1..9612cab 100644 (file)
@@ -11,7 +11,7 @@
 #   Craig Barratt  <cbarratt@users.sourceforge.net>
 #
 # COPYRIGHT
-#   Copyright (C) 2002  Craig Barratt
+#   Copyright (C) 2002-2003  Craig Barratt
 #
 #   This program is free software; you can redistribute it and/or modify
 #   it under the terms of the GNU General Public License as published by
@@ -29,7 +29,7 @@
 #
 #========================================================================
 #
-# Version 1.6.0_CVS, released 10 Dec 2002.
+# Version 2.1.0, released 20 Jun 2004.
 #
 # See http://backuppc.sourceforge.net.
 #
@@ -41,7 +41,7 @@ use strict;
 use BackupPC::View;
 use BackupPC::Xfer::RsyncFileIO;
 
-use vars qw( $RsyncLibOK );
+use vars qw( $RsyncLibOK $RsyncLibErr );
 
 BEGIN {
     eval "use File::RsyncP;";
@@ -50,8 +50,17 @@ BEGIN {
         # Rsync module doesn't exist.
         #
         $RsyncLibOK = 0;
+        $RsyncLibErr = "File::RsyncP module doesn't exist";
     } else {
-        $RsyncLibOK = 1;
+       #
+       # Note: also update configure.pl when this version number is changed!
+       #
+        if ( $File::RsyncP::VERSION < 0.51 ) {
+            $RsyncLibOK = 0;
+            $RsyncLibErr = "File::RsyncP module version too old: need 0.50";
+        } else {
+            $RsyncLibOK = 1;
+        }
     }
 };
 
@@ -68,6 +77,20 @@ sub new
         hostIP    => "",
         shareName => "",
         badFiles  => [],
+
+       #
+       # Various stats
+       #
+        byteCnt         => 0,
+       fileCnt         => 0,
+       xferErrCnt      => 0,
+       xferBadShareCnt => 0,
+       xferBadFileCnt  => 0,
+       xferOK          => 0,
+
+       #
+       # User's args
+       #
         %$args,
     }, $class;
 
@@ -93,133 +116,206 @@ sub start
     my($t) = @_;
     my $bpc = $t->{bpc};
     my $conf = $t->{conf};
-    my(@fileList, @rsyncClientCmd, $logMsg, $incrDate);
+    my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
+       $incrDate, $argList, $fioArgs);
+
+    #
+    # We add a slash to the share name we pass to rsync
+    #
+    ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{/};
 
     if ( $t->{type} eq "restore" ) {
-       # TODO
-        #push(@rsyncClientCmd, split(/ +/, $c o n f->{RsyncClientRestoreCmd}));
-        $logMsg = "restore not supported for $t->{shareName}";
-       #
-       # restores are considered to work unless we see they fail
-       # (opposite to backups...)
-       #
-       $t->{xferOK} = 1;
+        $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
+       $rsyncArgs = $conf->{RsyncRestoreArgs};
+       my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
+       $remoteDir    =~ s{//+}{/}g;
+        $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
+       $fioArgs = {
+           client   => $t->{bkupSrcHost},
+           share    => $t->{bkupSrcShare},
+           viewNum  => $t->{bkupSrcNum},
+           fileList => $t->{fileList},
+       };
+        $logMsg = "restore started below directory $t->{shareName}"
+               . " to host $t->{host}";
     } else {
        #
        # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
-       # into a hash of arrays of files.  NOT IMPLEMENTED YET.
+       # into a hash of arrays of files, and $conf->{RsyncShareName}
+       # to an array
        #
-       $conf->{RsyncShareName} = [ $conf->{RsyncShareName} ]
-                       unless ref($conf->{RsyncShareName}) eq "ARRAY";
-       foreach my $param qw(BackupFilesOnly BackupFilesExclude) {
-           next if ( !defined($conf->{$param}) );
-           if ( ref($conf->{$param}) eq "ARRAY" ) {
-               $conf->{$param} = {
-                       $conf->{RsyncShareName}[0] => $conf->{$param}
-               };
-           } elsif ( ref($conf->{$param}) eq "HASH" ) {
-               # do nothing
-           } else {
-               $conf->{$param} = {
-                       $conf->{RsyncShareName}[0] => [ $conf->{$param} ]
-               };
-           }
-       }
+       $bpc->backupFileConfFix($conf, "RsyncShareName");
+
+        if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
+            my(@inc, @exc, %incDone, %excDone);
+            foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
+                #
+                # If the user wants to just include /home/craig, then
+                # we need to do create include/exclude pairs at
+                # each level:
+                #     --include /home --exclude /*
+                #     --include /home/craig --exclude /home/*
+                #
+                # It's more complex if the user wants to include multiple
+                # deep paths.  For example, if they want /home/craig and
+                # /var/log, then we need this mouthfull:
+                #     --include /home --include /var --exclude /*
+                #     --include /home/craig --exclude /home/*
+                #     --include /var/log --exclude /var/*
+                #
+                # To make this easier we do all the includes first and all
+                # of the excludes at the end (hopefully they commute).
+                #
+                $file =~ s{/$}{};
+                $file = "/$file";
+                $file =~ s{//+}{/}g;
+               if ( $file eq "/" ) {
+                   #
+                   # This is a special case: if the user specifies
+                   # "/" then just include it and don't exclude "/*".
+                   #
+                    push(@inc, $file) if ( !$incDone{$file} );
+                   next;
+               }
+                my $f = "";
+                while ( $file =~ m{^/([^/]*)(.*)} ) {
+                    my $elt = $1;
+                    $file = $2;
+                    if ( $file eq "/" ) {
+                        #
+                        # preserve a tailing slash
+                        #
+                        $file = "";
+                        $elt = "$elt/";
+                    }
+                    push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
+                    $excDone{"$f/*"} = 1;
+                    $f = "$f/$elt";
+                    push(@inc, $f) if ( !$incDone{$f} );
+                    $incDone{$f} = 1;
+                }
+            }
+            foreach my $file ( @inc ) {
+                push(@fileList, "--include=$file");
+            }
+            foreach my $file ( @exc ) {
+                push(@fileList, "--exclude=$file");
+            }
+        }
         if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
             foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
             {
+                #
+                # just append additional exclude lists onto the end
+                #
                 push(@fileList, "--exclude=$file");
             }
         }
-        if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
-            foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
-                push(@fileList, $file);
-            }
-        } else {
-           push(@fileList, ".");
-        }
-       push(@rsyncClientCmd, split(/ +/, $conf->{RsyncClientCmd}));
         if ( $t->{type} eq "full" ) {
-            $logMsg = "full backup started for directory $t->{shareName}";
+           if ( $t->{partialNum} ) {
+               $logMsg = "full backup started for directory $t->{shareName};"
+                       . " updating partial $t->{partialNum}";
+           } else {
+               $logMsg = "full backup started for directory $t->{shareName}";
+           }
         } else {
-            $incrDate = $bpc->timeStampISO($t->{lastFull} - 3600, 1);
+            $incrDate = $bpc->timeStamp($t->{lastFull} - 3600, 1);
             $logMsg = "incr backup started back to $incrDate for directory"
                     . " $t->{shareName}";
         }
-       $t->{xferOK} = 0;
-    }
-    #
-    # Merge variables into @rsyncClientCmd
-    #
-    my $vars = {
-        host      => $t->{host},
-        hostIP    => $t->{hostIP},
-        shareName => $t->{shareName},
-        rsyncPath => $conf->{RsyncClientPath},
-        sshPath   => $conf->{SshPath},
-    };
-    my @cmd = @rsyncClientCmd;
-    @rsyncClientCmd = ();
-    foreach my $arg ( @cmd ) {
-       next if ( $arg =~ /^\s*$/ );
-       if ( $arg =~ /^\$fileList(\+?)/ ) {
-           my $esc = $1 eq "+";
-           foreach $arg ( @fileList ) {
-               $arg = $bpc->shellEscape($arg) if ( $esc );
-               push(@rsyncClientCmd, $arg);
-           }
-       } elsif ( $arg =~ /^\$argList(\+?)/ ) {
-           my $esc = $1 eq "+";
-           foreach $arg ( (@{$conf->{RsyncArgs}},
-                           @{$conf->{RsyncClientArgs}}) ) {
-               $arg = $bpc->shellEscape($arg) if ( $esc );
-               push(@rsyncClientCmd, $arg);
-           }
-       } else {
-           $arg =~ s{\$(\w+)(\+?)}{
-               defined($vars->{$1})
-                   ? ($2 eq "+" ? $bpc->shellEscape($vars->{$1}) : $vars->{$1})
-                   : "\$$1"
-           }eg;
-           push(@rsyncClientCmd, $arg);
-       }
+        
+        #
+        # A full dump is implemented with --ignore-times: this causes all
+        # files to be checksummed, even if the attributes are the same.
+        # That way all the file contents are checked, but you get all
+        # the efficiencies of rsync: only files deltas need to be
+        # transferred, even though it is a full dump.
+        #
+       $rsyncArgs = $conf->{RsyncArgs};
+       $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList );
+        $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
+                                    if ( $t->{type} eq "full" );
+       $rsyncClientCmd = $conf->{RsyncClientCmd};
+        $argList = ['--server', '--sender', @$rsyncArgs,
+                              '.', $t->{shareNameSlash}];
+       $fioArgs = {
+           client     => $t->{client},
+           share      => $t->{shareName},
+           viewNum    => $t->{lastFullBkupNum},
+            partialNum => $t->{partialNum},
+       };
     }
 
     #
-    # A full dump is implemented with --ignore-times: this causes all
-    # files to be checksummed, even if the attributes are the same.
-    # That way all the file contents are checked, but you get all
-    # the efficiencies of rsync: only files deltas need to be
-    # transferred, even though it is a full dump.
+    # Merge variables into $rsyncClientCmd
     #
-    my $rsyncArgs = $conf->{RsyncArgs};
-    $rsyncArgs = [@$rsyncArgs, "--ignore-times"] if ( $t->{type} eq "full" );
+    my $args = {
+       host      => $t->{host},
+       hostIP    => $t->{hostIP},
+       client    => $t->{client},
+       shareName => $t->{shareName},
+       shareNameSlash => $t->{shareNameSlash},
+       rsyncPath => $conf->{RsyncClientPath},
+       sshPath   => $conf->{SshPath},
+       argList   => $argList,
+    };
+    $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args);
 
     #
     # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
     # module, which handles all the special BackupPC file storage
     # (compression, mangling, hardlinks, special files, attributes etc).
     #
+    $t->{rsyncClientCmd} = $rsyncClientCmd;
     $t->{rs} = File::RsyncP->new({
-       logLevel   => $conf->{RsyncLogLevel},
-       rsyncCmd   => \@rsyncClientCmd,
-       rsyncArgs  => $rsyncArgs,
-       logHandler => sub {
-                         my($str) = @_;
-                         $str .= "\n";
-                         $t->{XferLOG}->write(\$str);
-                     },
-       fio        => BackupPC::Xfer::RsyncFileIO->new({
+       logLevel     => $t->{logLevel} || $conf->{RsyncLogLevel},
+       rsyncCmd     => sub {
+                           $bpc->verbose(0);
+                           $bpc->cmdExecOrEval($rsyncClientCmd, $args);
+                       },
+       rsyncCmdType => "full",
+       rsyncArgs    => $rsyncArgs,
+       timeout      => $conf->{ClientTimeout},
+       doPartial    => defined($t->{partialNum}) ? 1 : undef,
+       logHandler   =>
+                sub {
+                    my($str) = @_;
+                    $str .= "\n";
+                    $t->{XferLOG}->write(\$str);
+                    if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) {
+                        #
+                        # Files with read errors (eg: region locked files
+                        # on WinXX) are filled with 0 by rsync.  Remember
+                        # them and delete them later.
+                        #
+                        my $badFile = $1;
+                        $badFile =~ s/^\/+//;
+                        push(@{$t->{badFiles}}, {
+                                share => $t->{shareName},
+                                file  => $badFile
+                            });
+                    }
+                },
+       pidHandler   => sub {
+                           $t->{pidHandler}(@_);
+                       },
+       fio          => BackupPC::Xfer::RsyncFileIO->new({
                            xfer       => $t,
                            bpc        => $t->{bpc},
                            conf       => $t->{conf},
-                           host       => $t->{host},
                            backups    => $t->{backups},
-                           logLevel   => $conf->{RsyncLogLevel},
+                           logLevel   => $t->{logLevel}
+                                              || $conf->{RsyncLogLevel},
+                           logHandler => sub {
+                                             my($str) = @_;
+                                             $str .= "\n";
+                                             $t->{XferLOG}->write(\$str);
+                                         },
+                           cacheCheckProb => $conf->{RsyncCsumCacheVerifyProb},
+                           %$fioArgs,
                      }),
     });
 
-    # TODO: alarm($conf->{SmbClientTimeout});
     delete($t->{_errStr});
 
     return $logMsg;
@@ -230,12 +326,30 @@ sub run
     my($t) = @_;
     my $rs = $t->{rs};
     my $conf = $t->{conf};
+    my($remoteSend, $remoteDir, $remoteDirDaemon);
 
+    alarm($conf->{ClientTimeout});
+    if ( $t->{type} eq "restore" ) {
+       $remoteSend       = 0;
+       ($remoteDir       = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
+       ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
+       $remoteDirDaemon  = $t->{shareNameSlash}
+                               if ( $t->{pathHdrDest} eq ""
+                                             || $t->{pathHdrDest} eq "/" );
+    } else {
+       $remoteSend      = 1;
+       $remoteDir       = $t->{shareNameSlash};
+       $remoteDirDaemon = ".";
+    }
     if ( $t->{XferMethod} eq "rsync" ) {
        #
        # Run rsync command
        #
-       $rs->remoteStart(1, $t->{shareName});
+       my $str = "Running: "
+               . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}})
+               . "\n";
+       $t->{XferLOG}->write(\$str);
+       $rs->remoteStart($remoteSend, $remoteDir);
     } else {
        #
        # Connect to the rsync server
@@ -243,16 +357,30 @@ sub run
        if ( defined(my $err = $rs->serverConnect($t->{hostIP},
                                             $conf->{RsyncdClientPort})) ) {
            $t->{hostError} = $err;
+           my $str = "Error connecting to rsync daemon at $t->{hostIP}"
+                   . ":$conf->{RsyncdClientPort}: $err\n";
+           $t->{XferLOG}->write(\$str);
            return;
        }
-       if ( defined(my $err = $rs->serverService($t->{shareName},
-                                                "craig", "xyz123", 0)) ) {
+       #
+       # Pass module name, and follow it with a slash if it already
+       # contains a slash; otherwise just keep the plain module name.
+       #
+       my $module = $t->{shareName};
+       $module = $t->{shareNameSlash} if ( $module =~ /\// );
+       if ( defined(my $err = $rs->serverService($module,
+                                             $conf->{RsyncdUserName},
+                                             $conf->{RsyncdPasswd},
+                                             $conf->{RsyncdAuthRequired})) ) {
+           my $str = "Error connecting to module $module at $t->{hostIP}"
+                   . ":$conf->{RsyncdClientPort}: $err\n";
+           $t->{XferLOG}->write(\$str);
            $t->{hostError} = $err;
            return;
        }
-       $rs->serverStart(1, ".");
+       $rs->serverStart($remoteSend, $remoteDirDaemon);
     }
-    my $error = $rs->go($t->{shareName});
+    my $error = $rs->go($t->{shareNameSlash});
     $rs->serverClose();
 
     #
@@ -261,40 +389,58 @@ sub run
     # $rs->{stats}{totalWritten}
     # $rs->{stats}{totalSize}
     #
-    # qw(byteCnt fileCnt xferErrCnt xferBadShareCnt xferBadFileCnt
-    #           xferOK hostAbort hostError lastOutputLine)
-    #
     my $stats = $rs->statsFinal;
     if ( !defined($error) && defined($stats) ) {
-       $t->{xferOK}  = 1;
+       $t->{xferOK} = 1;
     } else {
-       $t->{xferOK}  = 0;
+       $t->{xferOK} = 0;
     }
-    $t->{byteCnt} = $stats->{childStats}{TotalFileSize}
-                 + $stats->{parentStats}{TotalFileSize};
-    $t->{fileCnt} = $stats->{childStats}{TotalFileCnt}
-                 + $stats->{parentStats}{TotalFileCnt};
+    $t->{xferErrCnt} = $stats->{remoteErrCnt}
+                    + $stats->{childStats}{errorCnt}
+                    + $stats->{parentStats}{errorCnt};
+    $t->{byteCnt}    = $stats->{childStats}{TotalFileSize}
+                    + $stats->{parentStats}{TotalFileSize};
+    $t->{fileCnt}    = $stats->{childStats}{TotalFileCnt}
+                    + $stats->{parentStats}{TotalFileCnt};
+    my $str = "Done: $t->{fileCnt} files, $t->{byteCnt} bytes\n";
+    $t->{XferLOG}->write(\$str);
     #
     # TODO: get error count, and call fio to get stats...
     #
     $t->{hostError} = $error if ( defined($error) );
 
-    return (
-       0,
-       $stats->{childStats}{ExistFileCnt}
-           + $stats->{parentStats}{ExistFileCnt},
-       $stats->{childStats}{ExistFileSize}
-           + $stats->{parentStats}{ExistFileSize},
-       $stats->{childStats}{ExistFileCompSize}
-           + $stats->{parentStats}{ExistFileCompSize},
-       $stats->{childStats}{TotalFileCnt}
-           + $stats->{parentStats}{TotalFileCnt},
-       $stats->{childStats}{TotalFileSize}
-           + $stats->{parentStats}{TotalFileSize},
-    );
+    if ( $t->{type} eq "restore" ) {
+       return (
+           $t->{fileCnt},
+           $t->{byteCnt},
+           0,
+           0
+       );
+    } else {
+       return (
+           0,
+           $stats->{childStats}{ExistFileCnt}
+               + $stats->{parentStats}{ExistFileCnt},
+           $stats->{childStats}{ExistFileSize}
+               + $stats->{parentStats}{ExistFileSize},
+           $stats->{childStats}{ExistFileCompSize}
+               + $stats->{parentStats}{ExistFileCompSize},
+           $stats->{childStats}{TotalFileCnt}
+               + $stats->{parentStats}{TotalFileCnt},
+           $stats->{childStats}{TotalFileSize}
+               + $stats->{parentStats}{TotalFileSize},
+       );
+    }
 }
 
-#        alarm($conf->{SmbClientTimeout});
+sub abort
+{
+    my($t, $reason) = @_;
+    my $rs = $t->{rs};
+
+    $rs->abort($reason);
+    return 1;
+}
 
 sub setSelectMask
 {
@@ -305,6 +451,7 @@ sub errStr
 {
     my($t) = @_;
 
+    return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
     return $t->{_errStr};
 }
 
@@ -312,7 +459,7 @@ sub xferPid
 {
     my($t) = @_;
 
-    return -1;
+    return ();
 }
 
 sub logMsg