r1830@llin: dpavlin | 2009-04-25 16:46:33 +0200

 implement --mirror http://www.example.com/
 to create local mirror of remote paths behind http

git-svn-id: svn+ssh://mjesec/home/dpavlin/svn/webpac2/trunk@1164 07558da8-63fa-0310-ba24-9fe276d99e06

diff --git a/run.pl b/run.pl
index f048ca9..8b3d666 100755 (executable)
--- a/run.pl
+++ b/run.pl
@@ -22,6 +22,7 @@ use File::Slurp;
 use Data::Dump qw/dump/;
 use Storable qw/dclone/;
 use Pod::Usage qw/pod2usage/;
+use LWP::Simple;
 
 use POSIX ":sys_wait_h"; # imports WNOHANG
 
@@ -102,6 +103,10 @@ Create just links
 
 Create merged index of databases which have links
 
+=item --mirror http://www.example.com
+
+Tries to download input path files from mirror URI
+
 =back
 
 =cut
@@ -122,6 +127,7 @@ my $marc_dump = 0;
 my $parallel = 0;
 my $only_links = 0;
 my $merge = 0;
+my $mirror;
 my $help;
 
 my $log = _new WebPAC::Common()->_get_logger();
@@ -143,6 +149,7 @@ GetOptions(
        "parallel=i" => \$parallel,
        "only-links!" => \$only_links,
        "merge" => \$merge,
+       "mirror=s" => \$mirror,
        "help" => \$help,
 );
 
@@ -403,6 +410,14 @@ foreach my $database ( sort keys %{ $config->databases } ) {
                        delete($input->{modify_file});
                }
 
+               if ( $mirror ) {
+                       my $path = $input->{path} || die "no input path in ",dump( $input );
+                       $log->info( "mirror ", $path, " ", -s $path, " bytes" );
+
+                       $log->warn( "$path not modified" )
+                               if mirror( "$mirror/$path", $path ) == RC_NOT_MODIFIED;
+               }
+
                my $input_db = new WebPAC::Input(
                        module => $input_module,
                        limit => $limit || $input->{limit},