X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=bin%2Fisi-download-results.pl;h=6bb37d4c4081385b5089a141fc5dce0bf0d9d16c;hb=27a60b583075e44cc866c9c3a8ab65e2143a4a6b;hp=dfab3640de7897f211ee490ed7ec1863c7091131;hpb=55bb3ece8bd54ba90e9003c6cbe0431baf3ac668;p=webpac2

diff --git a/bin/isi-download-results.pl b/bin/isi-download-results.pl
index dfab364..6bb37d4 100755
--- a/bin/isi-download-results.pl
+++ b/bin/isi-download-results.pl
@@ -12,23 +12,25 @@ use Text::Unaccent;
 # http://images.isiknowledge.com/WOK46/help/WOS/h_advanced_examples.html
 
 our $q = 'AD=Croatia';
-my $range_size = 500;
+$q = 'AU=BRATKO, D';
+#$q = 'AD=(croat* OR hrvat*)';
+
+my $range_size = 100;
 my $overlap    = 3; # between previous and this range
-my $results = 0;
+
+my $results = 1;
 my $citations = 0;
+my $cited_reference = 0; # html tables
+my $citing_articles = 0; # as many files as cited articles
+
 my $cites_by_year = 0;
 
 my $max_cites = 5000; # ISI limit to get cites
 
-if ( 0 ) {
-	$q = 'TS=psychology AND AD=Croatia';
-	$range_size = 50;
-	$overlap    = 0;
-	$max_cites  = 50;
-}
-
 $q = unac_string( 'utf-8', join(' ', @ARGV) ) if @ARGV;
 
+warn "QUERY: $q\n";
+
 our $mech = WWW::Mechanize->new(
 	autocheck => 0, # it dies in reference download with it!
 	cookie_jar => undef,
@@ -54,7 +56,7 @@ sub save_mech {
 }
 
 warn "# get session";
-$mech->get( 'http://isiknowledge.com/?DestApp=WOS' );
+$mech->get( 'http://www.webofknowledge.com/?DestApp=WOS' );
 save_mech;
 
 sub search {
@@ -92,24 +94,28 @@ sub get_results {
 		my $to = $from + $range_size;
 
 		warn "# submit_form results $from - $to\n";
+		save_mech;
 
 		$mech->submit_form(
-			form_name => 'summary_output_form',
+			form_name => 'output_form',
 			fields => {
-				record_select_type => 'range',
+				'value(record_select_type)' => 'range',
+				markFrom => $from,
+				markTo => $to,
+
 				mark_from => $from,
 				mark_to => $to,
 				mark_id => 'WOS',
 
-				qo_fields => 'fullrecord',
-				citedref => 'citedref',
+				fields_selection => 'ACCESSION_NUM FUNDING SUBJECT_CATEGORY JCR_CATEGORY LANG IDS PAGEC SABBR CITREFC ISSN PUBINFO KEYWORDS CITTIMES ADDRS CONFERENCE_SPONSORS DOCTYPE ABSTRACT CONFERENCE_INFO SOURCE TITLE AUTHORS',
+				filters	=> 'ACCESSION_NUM FUNDING SUBJECT_CATEGORY JCR_CATEGORY LANG IDS PAGEC SABBR CITREFC ISSN PUBINFO KEYWORDS CITTIMES ADDRS CONFERENCE_SPONSORS DOCTYPE ABSTRACT CONFERENCE_INFO SOURCE TITLE AUTHORS CITREF',
+				fullrec_fields_option => 'CITREF',
 
-				save_options => 'plain_text',
+				save_options => 'fieldtagged',
+				format => 'saveToFile',
 
-				fields => 'Full',
-				format => 'save',
 			},
-			button => 'save',
+			button => 'saveToFile',
 		);
 		save_mech;
 
@@ -119,19 +125,25 @@ sub get_results {
 			last;
 		}
 
+		if ( $mech->content !~ m{Please wait while your request is processed} ) {
+			warn "WARNING: expecting processing request";
+		}
+
 
 		my $path = "/tmp/isi.$q.$from-$to";
 		$path .= '.' . $desc if $desc;
 
 		warn "save $from - $to into $path\n";
-		$mech->follow_link( url_regex => qr/save_file/ );
+		$mech->submit_form(
+			form_name => 'etsForm',
+		);
 		save_mech $path;
 
 		$from += $range_size - $overlap;
 
 		$mech->back;
 		$mech->back;
-		#save_mech;
+		save_mech;
 	}
 }
 
@@ -212,13 +224,37 @@ sub years {
 	return $years;
 }
 
-search;
-years;
-get_results 'results' if $results;
+
+our $page = 1;
+sub next_page {
+	$page++;
+	warn "next_page $page\n";
+
+	$mech->submit_form(
+		form_name => 'summary_navigation',
+		fields => {
+			'page' => $page,
+		},
+	);
+
+	save_mech;
+
+	$mech->form_name( 'summary_navigation' );
+	my $is_next_page = $mech->value('page') == $page;
+	warn "no next_page" unless $is_next_page;
+	return $is_next_page;
+}
+
+if ( $results ) {
+	search;
+	years;
+	get_results;
+}
 
 if ( $citations ) {
 
 	citations;
+	years unless @ranges;
 
 	do {
 		my $part;
@@ -232,3 +268,74 @@ if ( $citations ) {
 	} while ( @ranges );
 
 }
+
+
+
+if ( $q =~ m{CA=(.+)} && $cited_reference ) {
+
+	my $CA = $1;
+
+	warn "# citated reference search";
+	$mech->follow_link( url_regex => qr/CitedReferenceSearch/ );
+	save_mech;
+
+
+	$mech->submit_form(
+		form_name => 'WOS_CitedReferenceSearch_input_form',
+		fields => {
+			'value(input1)' => $CA,
+		},
+	);
+
+	my $page = 1;
+	my $records = $1 if $mech->content =~ m/(\d+)\s+records/;
+	warn "# found $records records\n";
+	my $last_span = 'fake';
+
+	while (1) {
+		save_mech "/tmp/isi.$q.citedref.$page";
+
+		last unless next_page();
+
+		if ( $mech->content =~ m/(\d+\s*-\s*(\d+))/ ) {
+			warn "span: $1\n";
+			last if $2 == $records;
+			last if $1 == $last_span;
+			$last_span = $1;
+		} elsif ( $page > 5 ) {
+			warn "ARTIFICALLY LIMITED TO 5 PAGES WITHOUT VALID SPAN!";
+			last;
+		}
+
+	}
+
+}
+
+if ( $q =~ m{CA=(.+)} && $citing_articles ) {
+
+	search;
+
+	my $orig_q = $q;
+	my $nr = 0;
+
+	do {
+
+	foreach my $link ( $mech->find_all_links( url_regex => qr/CitingArticles.do/ ) ) {
+		$nr++;
+		warn "link $nr\n";
+		$mech->get( $link->url );
+		save_mech;
+		$q = $orig_q . '.citing_article.' . $nr;
+		get_results;
+		$mech->back;
+		$mech->back;
+
+		#last if $nr > 3; # FIXME only for development
+	}
+
+	} while next_page;
+
+	$q = $orig_q;
+}
+
+warn "OVER\n";