* Changes in preparation for 3.2.0beta1...

[BackupPC.git] / lib / BackupPC / Xfer / Ftp.pm

diff --git a/lib/BackupPC/Xfer/Ftp.pm b/lib/BackupPC/Xfer/Ftp.pm
index 26f8510..d6e1955 100644 (file)
--- a/lib/BackupPC/Xfer/Ftp.pm
+++ b/lib/BackupPC/Xfer/Ftp.pm
@@ -31,13 +31,12 @@
 #
 #========================================================================
 #
 #
 #========================================================================
 #

-# Unreleased, planned release in 3.2 (or 3.1.1)
+# Version 3.2.0beta1, released 5 Jan 2010.

 #
 # See http://backuppc.sourceforge.net.
 #
 #========================================================================
 
 #
 # See http://backuppc.sourceforge.net.
 #
 #========================================================================
 

-

 package BackupPC::Xfer::Ftp;
 
 use strict;
 package BackupPC::Xfer::Ftp;
 
 use strict;


@@ -168,7 +167,7 @@ sub start
     # Convert the encoding type of the names if at all possible
     #
     from_to( $args->{shareName}, "utf8", $conf->{ClientCharset} )
     # Convert the encoding type of the names if at all possible
     #
     from_to( $args->{shareName}, "utf8", $conf->{ClientCharset} )

-        if ( $conf->{ClientCharset} ne "" );
+                                if ( $conf->{ClientCharset} ne "" );

 
     #
     # Collect FTP configuration arguments and translate them for
 
     #
     # Collect FTP configuration arguments and translate them for


@@ -191,6 +190,7 @@ sub start
         $t->{xferErrCnt}++;
         return;
     }
         $t->{xferErrCnt}++;
         return;
     }

+    $t->logWrite("Connected to $args->{Host}\n", 2);

 
     #
     # Log in to the ftp server and set appropriate path information.
 
     #
     # Log in to the ftp server and set appropriate path information.


@@ -202,6 +202,7 @@ sub start
         $t->{xferErrCnt}++;
         return;
     }
         $t->{xferErrCnt}++;
         return;
     }

+    $t->logWrite("Login successful to $conf->{FtpUserName}\@$args->{Host}\n", 2);

 
     undef $@;
     eval { $t->{ftp}->binary(); };
 
     undef $@;
     eval { $t->{ftp}->binary(); };


@@ -211,6 +212,7 @@ sub start
         $t->{xferErrCnt}++;
         return;
     }
         $t->{xferErrCnt}++;
         return;
     }

+    $t->logWrite("Binary command successful\n", 2);

 
     undef $@;
     eval { $t->{shareName} =~ m/^\.?$/ || $t->{ftp}->cwd( $t->{shareName} ); };
 
     undef $@;
     eval { $t->{shareName} =~ m/^\.?$/ || $t->{ftp}->cwd( $t->{shareName} ); };


@@ -220,6 +222,7 @@ sub start
         $t->{xferErrCnt}++;
         return;
     }
         $t->{xferErrCnt}++;
         return;
     }

+    $t->logWrite("Set cwd to $t->{shareName}\n", 2);

 
     undef $@;
     eval { $t->{sharePath} = $t->{ftp}->pwd(); };
 
     undef $@;
     eval { $t->{sharePath} = $t->{ftp}->pwd(); };


@@ -229,23 +232,26 @@ sub start
         $t->{xferErrCnt}++;
         return;
     }
         $t->{xferErrCnt}++;
         return;
     }

+    $t->logWrite("Pwd returned as $t->{sharePath}\n", 2);

 
     #
     # log the beginning of action based on type
     #
     if ( $t->{type} eq 'restore' ) {
 
     #
     # log the beginning of action based on type
     #
     if ( $t->{type} eq 'restore' ) {

-        $logMsg = "restore started on directory $t->{shareName}";
+        $logMsg = "ftp restore for host $t->{host} started on directory "
+          . "$t->{shareName}\n";

 
     } elsif ( $t->{type} eq 'full' ) {
 
     } elsif ( $t->{type} eq 'full' ) {

-        $logMsg = "full backup started on directory $t->{shareName}";
+        $logMsg = "ftp full backup for host $t->{host} started on directory "
+          . "$t->{shareName}\n";

 
     } elsif ( $t->{type} eq 'incr' ) {
 
     } elsif ( $t->{type} eq 'incr' ) {

-

         $incrDate = $bpc->timeStamp( $t->{incrBaseTime} - 3600, 1 );
         $incrDate = $bpc->timeStamp( $t->{incrBaseTime} - 3600, 1 );

-        $logMsg = "incremental backup started back to $incrDate" .
-            " (backup #$t->{incrBaseBkupNum}) for directory" . "
-            $t->{shareName}";
+        $logMsg = "ftp incremental backup for $t->{host} started back to "
+          . "$incrDate (backup #$t->{incrBaseBkupNum}) for directory "
+          . "$t->{shareName}\n";

     }
     }

+    $t->logWrite($logMsg, 1);

 
     #
     # call the recursive function based on the type of action
 
     #
     # call the recursive function based on the type of action


@@ -253,17 +259,20 @@ sub start
     if ( $t->{type} eq 'restore' ) {
 
         $t->restore();
     if ( $t->{type} eq 'restore' ) {
 
         $t->restore();

-        $logMsg = "Restore of $args->{Host} complete";
+        $logMsg = "Restore of $t->{host} "
+                . ($t->{xferOK} ? "complete" : "failed");

 
     } elsif ( $t->{type} eq 'incr' ) {
 
         $t->backup();
 
     } elsif ( $t->{type} eq 'incr' ) {
 
         $t->backup();

-        $logMsg = "Incremental backup of $args->{Host} complete";
+        $logMsg = "Incremental backup of $t->{host} "
+                . ($t->{xferOK} ? "complete" : "failed");

 
     } elsif ( $t->{type} eq 'full' ) {
 
         $t->backup();
 
     } elsif ( $t->{type} eq 'full' ) {
 
         $t->backup();

-        $logMsg = "Full backup of $args->{Host} complete";
+        $logMsg = "Full backup of $t->{host} "
+                . ($t->{xferOK} ? "complete" : "failed");

     }
 
     delete $t->{_errStr};
     }
 
     delete $t->{_errStr};


@@ -297,8 +306,8 @@ sub run
         return ( $t->{fileCnt}, $t->{byteCnt}, 0, 0 );
 
     } else {
         return ( $t->{fileCnt}, $t->{byteCnt}, 0, 0 );
 
     } else {

-        return \( $tarErrs,      $nFilesExist, $sizeExist,
-                  $sizeExistCom, $nFilesTotal, $sizeTotal );
+        return ( $tarErrs,      $nFilesExist, $sizeExist,
+                 $sizeExistCom, $nFilesTotal, $sizeTotal );

     }
 }
 
     }
 }
 


@@ -488,6 +497,7 @@ sub backup
         $t->{xferErrCnt}++;
         return;
     }
         $t->{xferErrCnt}++;
         return;
     }

+    $t->logWrite("Created output directory $OutDir\n", 3);

 
     #
     # determine the filetype of the shareName and back it up
 
     #
     # determine the filetype of the shareName and back it up


@@ -535,15 +545,13 @@ sub getFTPArgs
         FirewallType => undef,                            # not used
         BlockSize    => $conf->{FtpBlockSize} || 10240,
         Port         => $conf->{FtpPort}      || 21,
         FirewallType => undef,                            # not used
         BlockSize    => $conf->{FtpBlockSize} || 10240,
         Port         => $conf->{FtpPort}      || 21,

-        Timeout      => $conf->{FtpTimeout}   || 120,
-        Debug        => 0,                                # do not touch
-        Passive      => 1,                                # do not touch
+        Timeout      => defined($conf->{FtpTimeout}) ? $conf->{FtpTimeout} : 120,
+        Debug        => $t->{logLevel} >= 10 ? 1 : 0,
+        Passive      => defined($conf->{FtpPassive}) ? $conf->{FtpPassive} : 1,          

         Hash         => undef,                            # do not touch
         Hash         => undef,                            # do not touch

-        LocalAddr    => "localhost",                      # do not touch

     };
 }
 
     };
 }
 

-

 #
 #   usage:
 #     $dirList = $t->remotels($path);
 #
 #   usage:
 #     $dirList = $t->remotels($path);


@@ -566,15 +574,22 @@ sub remotels
 
     my ( $dirContents, $remoteDir, $f );
 
 
     my ( $dirContents, $remoteDir, $f );
 

+    $remoteDir = [];

     undef $@;
     undef $@;

+    $t->logWrite("remotels: about to list $path\n", 4);

     eval {
         $dirContents = ( $path =~ /^\.?$/ ) ? $ftp->dir()
                                             : $ftp->dir("$path/");
     };
     if ($@) {
         $t->{xferErrCnt}++;
     eval {
         $dirContents = ( $path =~ /^\.?$/ ) ? $ftp->dir()
                                             : $ftp->dir("$path/");
     };
     if ($@) {
         $t->{xferErrCnt}++;

+        $t->logWrite("remotels: can't retrieve remote directory contents of $path: $!\n", 1);

         return "can't retrieve remote directory contents of $path: $!";
     }
         return "can't retrieve remote directory contents of $path: $!";
     }

+    if ( $t->{logLevel} >= 4 ) {
+        my $str = join("\n", @$dirContents);
+        $t->logWrite("remotels: got dir() result:\n$str\n", 4);
+    }

 
     foreach my $info ( @{parse_dir($dirContents)} ) {
 
 
     foreach my $info ( @{parse_dir($dirContents)} ) {
 


@@ -586,8 +601,11 @@ sub remotels
             mode  => $info->[4],
         };
 
             mode  => $info->[4],
         };
 

+        $t->logWrite("remotels: adding name $f->{name}, type $f->{type}, size $f->{size}, mode $f->{mode}\n", 4);
+

         $f->{utf8name} = $f->{name};
         $f->{utf8name} = $f->{name};

-        from_to( $f->{utf8name}, $conf->{ClientCharset}, "utf8" );
+        from_to( $f->{utf8name}, $conf->{ClientCharset}, "utf8" )
+                                if ( $conf->{ClientCharset} ne "" );

 
        $f->{fullName} = "$t->{sharePath}/$path/$f->{name}";
        $f->{fullName} =~ s/\/+/\//g;
 
        $f->{fullName} = "$t->{sharePath}/$path/$f->{name}";
        $f->{fullName} =~ s/\/+/\//g;


@@ -709,7 +727,8 @@ sub handleSymFile
     };
 
     $f->{utf8name} = $fSym->{name};
     };
 
     $f->{utf8name} = $fSym->{name};

-    from_to( $f->{utf8name}, $conf->{ClientCharset}, "utf8" );
+    from_to( $f->{utf8name}, $conf->{ClientCharset}, "utf8" )
+                            if ( $conf->{ClientCharset} ne "" );

 
     $f->{relPath}  = $fSym->{relPath};
     $f->{fullName} = "$t->{shareName}/$fSym->{relPath}/$fSym->{name}";
 
     $f->{relPath}  = $fSym->{relPath};
     $f->{fullName} = "$t->{shareName}/$fSym->{relPath}/$fSym->{name}";


@@ -755,9 +774,17 @@ sub handleDir
         }
     }
 
         }
     }
 

-    $attrib    = BackupPC::Attrib->new( { compress => $t->{Compress} } );
+    $t->logWrite("handleDir: dir->relPath = $dir->{relPath}, OutDir = $OutDir\n", 4);
+
+    $attrib    = BackupPC::Attrib->new( { compress => $t->{compress} } );

     $remoteDir = $t->remotels( $dir->{relPath} );
 
     $remoteDir = $t->remotels( $dir->{relPath} );
 

+    if ( ref($remoteDir) ne 'ARRAY' ) {
+        $t->logWrite("handleDir failed: $remoteDir\n", 1);
+        $t->logFileAction( "fail", $dir->{utf8name}, $dir );
+        return;
+    }
+

     if ( $t->{type} eq "incr" ) {
         $localDir  = $view->dirAttrib( $t->{incrBaseBkupNum},
                                        $t->{shareName}, $dir->{relPath} );
     if ( $t->{type} eq "incr" ) {
         $localDir  = $view->dirAttrib( $t->{incrBaseBkupNum},
                                        $t->{shareName}, $dir->{relPath} );


@@ -828,7 +855,7 @@ sub handleDir
     my $data     = $attrib->writeData();
 
     $poolWrite = BackupPC::PoolWrite->new( $bpc, $fileName, length($data),
     my $data     = $attrib->writeData();
 
     $poolWrite = BackupPC::PoolWrite->new( $bpc, $fileName, length($data),

-                                           $t->{Compress} );
+                                           $t->{compress} );

     $poolWrite->write( \$data );
     ( $exists, $digest, $outSize, $errs ) = $poolWrite->close();
 
     $poolWrite->write( \$data );
     ( $exists, $digest, $outSize, $errs ) = $poolWrite->close();
 


@@ -867,13 +894,12 @@ sub handleFile
     }
 
     my $attribInfo = {
     }
 
     my $attribInfo = {

-                       type  => BPC_FTYPE_FILE,
-                       mode  => $f->{mode},
-                       uid   => undef,            # unsupported
-                       gid   => undef,            # unsupported
-                       size  => $f->{size},
-                       mtime => $f->{mtime},
-                     };
+        %$f,
+        type  => BPC_FTYPE_FILE,
+        uid   => undef,            # unsupported
+        gid   => undef,            # unsupported
+    };
+    delete $attribInfo->{utf8name}; # unused value

 
     #
     # If this is a full backup or the file has changed on the host,
 
     #
     # If this is a full backup or the file has changed on the host,


@@ -882,16 +908,15 @@ sub handleFile
     undef $@;
     eval { tie ( *FTP, 'Net::FTP::RetrHandle', $ftp, $f->{fullName} ); };
     if ( !*FTP || $@ ) {
     undef $@;
     eval { tie ( *FTP, 'Net::FTP::RetrHandle', $ftp, $f->{fullName} ); };
     if ( !*FTP || $@ ) {

-        $t->handleFileAction( "fail", $attribInfo );
+        $t->logFileAction( "fail", $f->{utf8name}, $attribInfo );

         $t->{xferBadFileCnt}++;
         $stats->{errCnt}++;
         return;
     }
 
         $t->{xferBadFileCnt}++;
         $stats->{errCnt}++;
         return;
     }
 

-    $poolFile  = $OutDir . "/" . $bpc->fileNameMangle( $f->{name} );
-    $poolWrite = BackupPC::PoolWrite->new( $bpc, $poolFile, $f->{size},
-                                           $bpc->{xfer}{compress} );
-
+    $poolFile    = $OutDir . "/" .  $bpc->fileNameMangle( $f->{name} );
+    $poolWrite   = BackupPC::PoolWrite->new( $bpc, $poolFile, $f->{size},
+                                           $t->{compress} );

     $localSize = 0;
 
     undef $@;
     $localSize = 0;
 
     undef $@;


@@ -905,6 +930,7 @@ sub handleFile
     if ( !*FTP || $@ || @$errs ) {
 
         $t->logFileAction( "fail", $f->{utf8name}, $attribInfo );
     if ( !*FTP || $@ || @$errs ) {
 
         $t->logFileAction( "fail", $f->{utf8name}, $attribInfo );

+        $t->logWrite("Unlinking($poolFile) because of error on close\n", 3);

         unlink($poolFile);
         $t->{xferBadFileCnt}++;
         $stats->{errCnt} += scalar @$errs;
         unlink($poolFile);
         $t->{xferBadFileCnt}++;
         $stats->{errCnt} += scalar @$errs;


@@ -916,7 +942,8 @@ sub handleFile
     #
     if ( $localSize != $f->{size} ) {
         $t->logFileAction( "fail", $f->{utf8name}, $attribInfo );
     #
     if ( $localSize != $f->{size} ) {
         $t->logFileAction( "fail", $f->{utf8name}, $attribInfo );

-        unklink($poolFile);
+        $t->logWrite("Unlinking($poolFile) because of size mismatch ($localSize vs $f->{size})\n", 3);
+        unlink($poolFile);

         $stats->{xferBadFileCnt}++;
         $stats->{errCnt}++;
         return;
         $stats->{xferBadFileCnt}++;
         $stats->{errCnt}++;
         return;


@@ -928,7 +955,12 @@ sub handleFile
     $attrib->set( $f->{utf8name}, $attribInfo );
     $t->logFileAction( $exists ? "pool" : "create",
                        $f->{utf8name}, $attribInfo );
     $attrib->set( $f->{utf8name}, $attribInfo );
     $t->logFileAction( $exists ? "pool" : "create",
                        $f->{utf8name}, $attribInfo );

-    print $newFilesFH "$digest $f->{size} $poolFile\n" unless $exists;
+
+    my $relPoolFile = $bpc->fileNameEltMangle( $t->{shareName} )
+                    . "/"
+                    . $bpc->fileNameMangle($attribInfo->{relPath});
+
+    print $newFilesFH "$digest $f->{size} $relPoolFile\n" unless $exists;

 
     #
     # Cumulate the stats
 
     #
     # Cumulate the stats


@@ -955,12 +987,13 @@ sub incrFileExistCheck
     my $view = $t->{view};
 
     my $oldAttribInfo = $view->fileAttrib( $t->{incrBaseBkupNum},
     my $view = $t->{view};
 
     my $oldAttribInfo = $view->fileAttrib( $t->{incrBaseBkupNum},

-                                       $t->{shareName}, $f->{relPath} );
+                                       $t->{shareName}, "/" . $f->{relPath} );

 
 

-    #print STDERR "*" x 50 . "\n";
-    #print STDERR "Old data:\n" . Dumper($oldAttribInfo);
-    #print STDERR "New data:\n" . Dumper($f);
-    #print STDERR "$f->{fullName}: $oldAttribInfo->{mtime} ?= $f->{mtime}, $oldAttribInfo->{size} ?= $f->{size}\n";
+    ##$t->logWrite( "Old attrib:\n" . Dumper($oldAttribInfo), 1 );
+    ##$t->logWrite( "New attrib:\n" . Dumper($f), 1 );
+    ##$t->logWrite( sprintf("%s: mtime %d vs %d, size %d vs %d\n", $f->{fullName},
+    ##              $oldAttribInfo->{mtime}, $f->{mtime},
+    ##              $oldAttribInfo->{size}, $f->{size}), 1);

 
     return ( $oldAttribInfo->{mtime} == $f->{mtime}
           && $oldAttribInfo->{size} == $f->{size} );
 
     return ( $oldAttribInfo->{mtime} == $f->{mtime}
           && $oldAttribInfo->{size} == $f->{size} );


@@ -983,9 +1016,11 @@ sub logFileAction
     $name  = "."   if ( $name  eq "" );
     $owner = "-/-" if ( $owner eq "/" );
 
     $name  = "."   if ( $name  eq "" );
     $owner = "-/-" if ( $owner eq "/" );
 

-    my $fileAction = sprintf( "  %-6s %1s%4o %9s %11.0f %s\n",
-                              $action, $type, $attrib->{mode} & 07777,
-                              $owner, $attrib->{size}, $name );
+    my $fileAction = sprintf(
+        "  %-6s %1s%4o %9s %11.0f %s\n",
+        $action, $type, $attrib->{mode} & 07777,
+        $owner, $attrib->{size}, $attrib->{relPath}
+    );

 
     return $t->logWrite( $fileAction, 1 );
 }
 
     return $t->logWrite( $fileAction, 1 );
 }