follow next links to crawl all data

[ILL-Zotero-RT] / zotero.pl

diff --git a/zotero.pl b/zotero.pl
index 241c832..2308e2f 100755 (executable)
--- a/zotero.pl
+++ b/zotero.pl
@@ -19,11 +19,20 @@ my $FETCH  = $ENV{FETCH}  || 0;
 my $db = CouchDB->new('10.60.0.95', 5984);
 eval { $db->put("z_$UserID") }; # create user database
 
-my $url = "https://api.zotero.org/users/$UserID/items?key=$key&format=atom&content=json&order=dateModified&sort=desc";
+my $url = "https://api.zotero.org/users/$UserID/items?format=atom&content=json&order=dateModified&sort=desc";
 
-my $file = "$UserID.atom";
+my $tree;
+my $ticket_items;
+my $items;
+
+restart:
 
-warn "# $url\n";
+$url .= '&key=' . $key;
+
+my $file = $UserID . '.' . md5_hex($url) . '.atom';
+$FETCH = 1 if ! -e $file;
+
+warn "# $url -> $file\n";
 if ( $FETCH && mirror( $url => $file ) == RC_NOT_MODIFIED ) {
        warn "not modified";
 #      exit 0;
@@ -32,11 +41,6 @@ if ( $FETCH && mirror( $url => $file ) == RC_NOT_MODIFIED ) {
 my $feed = XMLin( $file );
 #warn "# feed ",dump($feed);
 
-my $tree;
-my $ticket_items;
-
-my $items;
-
 foreach my $entry ( keys %{ $feed->{entry} } ) {
        warn "# entry $entry ",dump($entry);
        my $id = $entry;
@@ -77,12 +81,19 @@ foreach my $entry ( keys %{ $feed->{entry} } ) {
 
        $items->{$id} = $item;
 
-       my $old_item = $db->get( "z_$UserID/$id" );
-       warn "# old_item ",dump($old_item);
        my $json_md5 = md5_hex encode_json $item;
        $item->{json_md5} = $json_md5;
-       if ( $old_item->{zapi_etag} ne $item->{zapi_etag} || $json_md5 ne $old_item->{json_md5} ) {
-               $item->{_rev} = $old_item->{_rev};
+
+       if ( my $old_item = eval { $db->get( "z_$UserID/$id" ) } ) {
+               warn "# old_item ",dump($old_item);
+
+               if ( $old_item->{zapi_etag} ne $item->{zapi_etag} || $json_md5 ne $old_item->{json_md5} ) {
+                       $item->{_rev} = $old_item->{_rev};
+                       $db->put( "z_$UserID/$id" => $item );
+               } else {
+                       warn "# unchanged";
+               }
+       } else {
                $db->put( "z_$UserID/$id" => $item );
        }
 }
@@ -90,6 +101,12 @@ foreach my $entry ( keys %{ $feed->{entry} } ) {
 delete $feed->{entry};
 warn "# feed without entry ",dump( $feed );
 
+if ( my @next = map { $_->{href} } grep { $_->{rel} eq 'next' && $_->{type} eq 'application/atom+xml' } @{ $feed->{link} } ) {
+       warn "## next ",dump(@next);
+       $url = $next[0];
+       goto restart;
+}
+
 warn "# tree ",dump( $tree );
 
 warn "# ticket_items ",dump( $ticket_items );