save results range_size by range_size

git-svn-id: svn+ssh://mjesec/home/dpavlin/svn/webpac2/trunk@1281 07558da8-63fa-0310-ba24-9fe276d99e06

diff --git a/bin/isi-download-results.pl b/bin/isi-download-results.pl
index 95bfd43..2cc88b6 100755 (executable)
--- a/bin/isi-download-results.pl
+++ b/bin/isi-download-results.pl
@@ -7,22 +7,21 @@ my $q = 'TS=psychology AND AD=Croatia';
 
 use WWW::Mechanize;
 use Data::Dump qw/dump/;
-use File::Slurp;
 
 my $mech = WWW::Mechanize->new(
        autocheck => 1,
        cookie_jar => undef,
 );
 
-my $step = 1;
+our $step = 0;
 
 sub save_mech {
-       my $mech = shift;
-       my $path = "/tmp/isi.$step.html";
-       write_file $path, $mech->content;
-       warn "# step $step ", -s $path;
-       $mech->dump_all;
+       my ( $mech, $path ) = @_;
        $step++;
+       $path ||= sprintf('/tmp/isi.%02d.%s', $step, $mech->{ct} =~ m{html}i ? 'html' : 'txt' );
+       $mech->save_content( $path );
+       warn "# [$step] $path ", -s $path, " ", $mech->ct;
+       $mech->dump_all;
 }
 
 warn "# get session";
@@ -47,29 +46,42 @@ $mech->follow_link( url_regex => qr/summary/ );
 save_mech $mech;
 
 my $from = 1;
-my $range_size = 10;
+my $range_size = 50;
 
-$mech->submit_form(
-       form_name => 'summary_output_form',
-       fields => {
-               record_select_type => 'range',
-               mark_from => $from,
-               mark_to => $from += $range_size,
-               mark_id => 'WOS',
+while ( $from ) {
 
-               qo_fields => 'fullrecord',
-               citedref => 'citedref',
+       my $to = $from + $range_size;
 
-               save_options => 'plain_text',
+       $mech->submit_form(
+               form_name => 'summary_output_form',
+               fields => {
+                       record_select_type => 'range',
+                       mark_from => $from,
+                       mark_to => $to,
+                       mark_id => 'WOS',
 
-               fields => 'Full',
-               format => 'save',
-       },
-       button => 'save',
-);
-save_mech $mech;
+                       qo_fields => 'fullrecord',
+                       citedref => 'citedref',
 
-warn "# save_file";
-$mech->follow_link( url_regex => qr/save_file/ );
-save_mech $mech;
+                       save_options => 'plain_text',
 
+                       fields => 'Full',
+                       format => 'save',
+               },
+               button => 'save',
+       );
+       save_mech $mech;
+
+       last if $mech->content =~ m{invalid API call};
+
+       warn "# save_file";
+       $mech->follow_link( url_regex => qr/save_file/ );
+       save_mech $mech => "/tmp/isi.$q.$from-$to.txt";
+
+       $from += $range_size;
+
+       $mech->back;
+       $mech->back;
+       #save_mech $mech;
+
+}