extract update to CouchDB to force update document

[ILL-Zotero-RT] / zotero.pl

diff --git a/zotero.pl b/zotero.pl
index 2546c34..afee591 100755 (executable)
--- a/zotero.pl
+++ b/zotero.pl
@@ -8,16 +8,42 @@ use JSON;
 use Data::Dump qw(dump);
 use RT::Client::REST;
 
+use CouchDB;
+use Digest::MD5 qw(md5_hex);
+
 my $UserID = $ENV{UserID} || die "usage: UserID=1234 key=abcd $0";
 my $key    = $ENV{key}    || die "key required";
 
 my $FETCH  = $ENV{FETCH}  || 0;
 
-my $url = "https://api.zotero.org/users/$UserID/items?key=$key&format=atom&content=json&order=dateModified&sort=desc";
+my $db = CouchDB->new('10.60.0.92', 5984);
+eval { $db->put("zotero_$UserID") }; # create user database
+eval {
+       local $/ = undef;
+       my $view = <DATA>;
+       warn "# create $view";
+       $db->put("zotero_$UserID/_design/zotero" => decode_json $view)
+};
+
+eval { $db->put("rt") }; # create RT database
+
+my @urls = map { "https://api.zotero.org/users/$UserID/$_?format=atom&content=json&order=dateModified&sort=desc" } qw( collections items );
+# we don't need to fetch tags since we can generate using CouchDB views
+
+my $url = shift @urls;
+
+my $tree;
+my $ticket_items;
+my $items;
+
+restart:
 
-my $file = "$UserID.atom";
+$url .= '&key=' . $key;
 
-warn "# $url\n";
+my $file = $UserID . '.' . md5_hex($url) . '.atom';
+$FETCH = 1 if ! -e $file;
+
+warn "# $url -> $file\n";
 if ( $FETCH && mirror( $url => $file ) == RC_NOT_MODIFIED ) {
        warn "not modified";
 #      exit 0;
@@ -26,44 +52,82 @@ if ( $FETCH && mirror( $url => $file ) == RC_NOT_MODIFIED ) {
 my $feed = XMLin( $file );
 #warn "# feed ",dump($feed);
 
-my $tree;
-my $ticket_items;
-
-my $items;
+sub link_to_id {
+       my $link = shift;
+       $link =~ s{.+/(items|collections)/}{}; # leave just ID
+       $link =~ s{\?.+}{};
+       return $link;
+}
 
 foreach my $entry ( keys %{ $feed->{entry} } ) {
        warn "# entry $entry ",dump($entry);
-       my $id = $entry;
-       $id =~ s{.+/items/}{}; # leave just ID
+       my $id = link_to_id $entry;
 
        my $item = $feed->{entry}->{$entry};
+       warn "# entry $entry ",dump($item);
+
+       foreach my $i ( 0 .. $#{ $item->{link} } ) {
+               my $link = $item->{link}->[$i];
+               warn "# link $id $i:",dump($link);
+
+               $item->{link}->[$i]->{key} = link_to_id $link->{href};
 
-       foreach my $link ( @{ $item->{link} } ) {
-               warn "# link $id ",dump($link);
                if ( $link->{rel} eq 'up' ) {
-                       my $up = $link->{href};
-                       $up =~ s{.+/items/}{};
-                       $up =~ s{\?.+}{};
-                       push @{ $tree->{$up} }, $id;
+                       push @{ $tree->{$key} }, $id;
                }
        }
 
-       if ( $item->{content}->{'zapi:type'} eq 'json' ) {
-               my $json = $item->{content}->{content};
-               warn "# $json\n";
-               $json = $item->{content} = decode_json $json;
-               warn "# json $id ", dump $json;
-
-               foreach my $tag ( @{ $json->{tags} } ) {
-                       $tag = $tag->{tag};
-                       warn "# tag $id $tag\n";
-                       next unless $tag =~ m/#(\d+)/;
-                       push @{ $ticket_items->{$1} }, $id;
+       if ( exists $item->{content} ) {
+               my $type = ( grep { exists $item->{content}->{$_} } qw(zapi:type type) )[0];
+               warn "# content has $type";
+
+               $item->{zapi}->{etag} = $item->{content}->{'zapi:etag'} if exists $item->{content}->{'zapi:etag'};
+
+               $type = $item->{zapi}->{type} = $item->{content}->{$type};
+
+               if ( $type =~ m/json/ ) {
+
+                       my $json = $item->{content}->{content};
+                       warn "# $json\n";
+                       $json = $item->{content} = decode_json $json;
+                       warn "# json $id ", dump $json;
+
+                       foreach my $tag ( @{ $json->{tags} } ) {
+                               $tag = $tag->{tag};
+                               warn "# tag $id $tag\n";
+                               next unless $tag =~ m/#(\d+)/; # XXX RT number in tag
+                               push @{ $ticket_items->{$1} }, $id;
+                       }
+
+               } else {
+                       warn "ERROR: $type not decoded!";
                }
        }
 
+       foreach my $zapi ( grep { m/^zapi:/ } keys %$item ) {
+               my $name = $zapi;
+               $name =~ s/^zapi://;
+               $item->{zapi}->{$name} = delete $item->{$zapi};
+       }
+
        $items->{$id} = $item;
 
+       $db->update( "zotero_$UserID/$id" => $item );
+
+}
+
+delete $feed->{entry};
+warn "# feed without entry ",dump( $feed );
+
+if ( my @next = map { $_->{href} } grep { $_->{rel} eq 'next' && $_->{type} eq 'application/atom+xml' } @{ $feed->{link} } ) {
+       warn "## next ",dump(@next);
+       $url = $next[0];
+       goto restart;
+}
+
+if ( $url = shift @urls ) {
+       warn "## next url $url";
+       goto restart;
 }
 
 warn "# tree ",dump( $tree );
@@ -83,6 +147,8 @@ foreach my $nr ( keys %$ticket_items ) {
        my $ticket = $rt->show(type => 'ticket', id => $nr);
        warn "# ticket $nr ",dump($ticket);
 
+       $db->update( "rt/$nr" => $ticket );
+
        if ( $ticket->{Queue} !~ m/ILL/i ) {
                warn "SKIP $ticket not in ILL queue!";
                next;
@@ -91,10 +157,13 @@ foreach my $nr ( keys %$ticket_items ) {
        foreach my $id ( @{ $ticket_items->{$nr} } ) {
                warn "# item $id ",dump( $items->{$id} );
 
-               $rt->comment( ticket_id => $nr, message => dump( $items->{$id} ) );
+#              $rt->comment( ticket_id => $nr, message => dump( $items->{$id} ) );
 
                last; # FIXME just first
 
        }
 
 }
+
+__DATA__
+{"_id":"_design/zotero","language":"javascript","views":{"itemType":{"map":"function(doc) {\n  emit(doc.zapi.itemType,1);\n}","reduce":"_count"},"updated":{"map":"function(doc) {\n  emit(doc.updated,1);\n}","reduce":"_count"},"tags":{"map":"function(doc) {\n  \n  doc.content.tags.forEach( function(v) {\n    emit(v, doc._id);\n  });\n}","reduce":"_count"},"link_up":{"map":"function(doc) {\n  if ( doc.link[1].rel == 'up' )\n  emit( doc.link[1].key, doc._id );\n}","reduce":"_count"},"year,publisher":{"map":"function(doc) {\n  if ( doc.zapi.year )\n  emit([doc.zapi.year, doc.content.publisher], 1);\n}","reduce":"_count"}}}