X-Git-Url: http://git.rot13.org/?p=ILL-Zotero-RT;a=blobdiff_plain;f=zotero.pl;h=2dda7841ea0fbbfdd8b325882cd8501d20728ea1;hp=efbf46839cba24c640490f48851f487f1e48fa42;hb=HEAD;hpb=d9289d0cc1231c2d096c63cee67b7fd120481ec8 diff --git a/zotero.pl b/zotero.pl index efbf468..2dda784 100755 --- a/zotero.pl +++ b/zotero.pl @@ -7,6 +7,7 @@ use XML::Simple; use JSON; use Data::Dump qw(dump); use RT::Client::REST; +use URI::Escape; use CouchDB; use Digest::MD5 qw(md5_hex); @@ -16,10 +17,21 @@ my $key = $ENV{key} || die "key required"; my $FETCH = $ENV{FETCH} || 0; -my $db = CouchDB->new('10.60.0.95', 5984); -eval { $db->put("z_$UserID") }; # create user database +my $db = CouchDB->new('10.60.0.92', 5984); +eval { $db->put("zotero_$UserID") }; # create user database +eval { + local $/ = undef; + my $view = ; + warn "# create $view"; + $db->put("zotero_$UserID/_design/zotero" => decode_json $view) +}; -my $url = "https://api.zotero.org/users/$UserID/items?format=atom&content=json&order=dateModified&sort=desc"; +eval { $db->put("rt") }; # create RT database + +my @urls = map { "https://api.zotero.org/users/$UserID/$_?format=atom&content=json&order=dateModified&sort=desc" } qw( collections items ); +# we don't need to fetch tags since we can generate using CouchDB views + +my $url = shift @urls; my $tree; my $ticket_items; @@ -32,51 +44,75 @@ $url .= '&key=' . $key; my $file = $UserID . '.' . md5_hex($url) . '.atom'; $FETCH = 1 if ! -e $file; -warn "# $url -> $file\n"; +warn "# mirror $FETCH $url -> $file\n"; if ( $FETCH && mirror( $url => $file ) == RC_NOT_MODIFIED ) { warn "not modified"; -# exit 0; } -my $feed = XMLin( $file ); -#warn "# feed ",dump($feed); +my $xml = XML::Simple->new(ForceArray => [ qw( entry ) ]); +my $feed = eval { $xml->XMLin( $file ) }; +if ( $! ) { + warn "ERROR $file $!\n"; + goto skip_url; +} +warn "# feed ",dump($feed); + +sub link_to_id { + my $link = shift; + $link =~ s{.+/(items|collections)/}{}; # leave just ID + $link =~ s{\?.+}{}; + return $link; +} + +my @collection_items; foreach my $entry ( keys %{ $feed->{entry} } ) { warn "# entry $entry ",dump($entry); - my $id = $entry; - $id =~ s{.+/items/}{}; # leave just ID + my $id = link_to_id $entry; + + push @collection_items, $id if $url =~ m{/collections/(\w+)/items}; my $item = $feed->{entry}->{$entry}; - warn "# entry $entry ",dump($item); + warn "# item $id $entry ",dump($item),$/; foreach my $i ( 0 .. $#{ $item->{link} } ) { my $link = $item->{link}->[$i]; warn "# link $id $i:",dump($link); - my $key = $link->{href}; - $key =~ s{.+/items/}{}; - $key =~ s{\?.+}{}; - $item->{link}->[$i]->{key} = $key; + $item->{link}->[$i]->{key} = link_to_id $link->{href}; if ( $link->{rel} eq 'up' ) { push @{ $tree->{$key} }, $id; + } elsif ( $link->{rel} eq 'self' && $link->{href} =~ m{/collections/} ) { + warn "# get items in this collection"; + push @urls, "$link->{href}/items?content=json"; } } - $item->{zapi}->{etag} = $item->{content}->{'zapi:etag'}; - $item->{zapi}->{type} = $item->{content}->{'zapi:type'}; + if ( exists $item->{content} ) { + my $type = ( grep { exists $item->{content}->{$_} } qw(zapi:type type) )[0]; + warn "# content has $type"; - if ( $item->{content}->{'zapi:type'} eq 'json' ) { - my $json = $item->{content}->{content}; - warn "# $json\n"; - $json = $item->{content} = decode_json $json; - warn "# json $id ", dump $json; + $item->{zapi}->{etag} = $item->{content}->{'zapi:etag'} if exists $item->{content}->{'zapi:etag'}; - foreach my $tag ( @{ $json->{tags} } ) { - $tag = $tag->{tag}; - warn "# tag $id $tag\n"; - next unless $tag =~ m/#(\d+)/; - push @{ $ticket_items->{$1} }, $id; + $type = $item->{zapi}->{type} = $item->{content}->{$type}; + + if ( $type =~ m/json/ ) { + + my $json = $item->{content}->{content}; + warn "# $json\n"; + $json = $item->{content} = decode_json $json; + warn "# json $id ", dump $json; + + foreach my $tag ( @{ $json->{tags} } ) { + $tag = $tag->{tag}; + warn "# tag $id $tag\n"; + next unless $tag =~ m/#(\d+)/; # XXX RT number in tag + push @{ $ticket_items->{$1} }, $id; + } + + } else { + warn "ERROR: $type not decoded!"; } } @@ -88,21 +124,17 @@ foreach my $entry ( keys %{ $feed->{entry} } ) { $items->{$id} = $item; - my $json_md5 = md5_hex encode_json $item; - $item->{json_md5} = $json_md5; + $db->modify( "zotero_$UserID/$id" => $item ); - if ( my $old_item = eval { $db->get( "z_$UserID/$id" ) } ) { - warn "# old_item ",dump($old_item); +} - if ( $old_item->{zapi}->{etag} ne $item->{zapi}->{etag} || $json_md5 ne $old_item->{json_md5} ) { - $item->{_rev} = $old_item->{_rev}; - $db->put( "z_$UserID/$id" => $item ); - } else { - warn "# unchanged"; - } - } else { - $db->put( "z_$UserID/$id" => $item ); - } +if ( @collection_items ) { + my $id = $1 if $url =~ m{/collections/(\w+)/items}; + $db->modify( "zotero_$UserID/$id" => sub { + my $doc = shift; + $doc->{x_meta}->{collection_items} = [ @collection_items ]; + return $doc; + }); } delete $feed->{entry}; @@ -114,6 +146,13 @@ if ( my @next = map { $_->{href} } grep { $_->{rel} eq 'next' && $_->{type} eq ' goto restart; } +skip_url: + +if ( $url = shift @urls ) { + warn "## next url $url"; + goto restart; +} + warn "# tree ",dump( $tree ); warn "# ticket_items ",dump( $ticket_items ); @@ -128,9 +167,54 @@ $rt->login(username => $ENV{RT_USER}, password => $ENV{RT_PASSWORD}); foreach my $nr ( keys %$ticket_items ) { - my $ticket = $rt->show(type => 'ticket', id => $nr); + my $ticket = eval { $rt->show(type => 'ticket', id => $nr) }; warn "# ticket $nr ",dump($ticket); + next unless $ticket; + + $ticket->{zotero_items} = $ticket_items->{$nr}; + + my $modified = $db->modify( "rt/$nr" => sub { + my $doc = shift; + $doc->{$_} = $ticket->{$_} foreach keys %$ticket; + return $doc; + }); + + warn "# modified ",dump($modified); + + # copy attachments to CouchDB (they never change, so do it just once + if ( my @attachment_ids = $rt->get_attachment_ids( id => $nr ) ) { + + warn "# get_attachment_ids = ",dump( @attachment_ids ); + my $doc = $db->get("rt/$nr"); + my @attachments; + + foreach my $attachment_id ( @attachment_ids ) { + my $attachment = $rt->get_attachment( parent_id => $nr, id => $attachment_id ); + if ( $attachment->{Filename} && $attachment->{ContentEncoding} eq 'base64' ) { + #$attachment->{Filename} ||= $attachment_id; + my $content = delete $attachment->{Content}; + if ( ! exists $doc->{_attachments}->{ $attachment->{Filename} } ) { + utf8::encode($content) || warn "utf8::encode error!"; + warn "# extracted ",length( $content ), " bytes"; + warn "## attachment ",dump( $attachment ); + my $url = sprintf 'rt/%d/%s?rev=%s', $nr, uri_escape($attachment->{Filename}), $modified->{rev}; +# $modified = $db->request( PUT => $url, $content, $attachment->{ContentType} ); + } + } + push @attachments, $attachment; + } + + + $db->modify( "rt/$nr" => sub { + my $doc = shift; + $doc->{attachments} = [ @attachments ]; + warn "## attachments on $nr = ", $#attachments + 1; + return $doc; + }) if @attachments; + + } + if ( $ticket->{Queue} !~ m/ILL/i ) { warn "SKIP $ticket not in ILL queue!"; next; @@ -141,8 +225,9 @@ foreach my $nr ( keys %$ticket_items ) { # $rt->comment( ticket_id => $nr, message => dump( $items->{$id} ) ); - last; # FIXME just first - } } + +__DATA__ +{"_id":"_design/zotero","views":{"link_up":{"map":"function(doc) {\n if ( doc.link[1].rel == 'up' )\n emit( doc.link[1].key, doc._id );\n}","reduce":"_count"},"year,publisher":{"map":"function(doc) {\n if ( doc.zapi.year )\n emit([doc.zapi.year, doc.content.publisher], 1);\n}","reduce":"_count"},"updated":{"map":"function(doc) {\n emit(doc.updated,1);\n}","reduce":"_count"},"itemType":{"map":"function(doc) {\n emit(doc.zapi.itemType,1);\n}","reduce":"_count"},"tags":{"map":"function(doc) {\n \n doc.content.tags.forEach( function(v) {\n emit(v, doc._id);\n });\n}","reduce":"_count"},"collection_items":{"map":"function(doc) {\n if ( doc.x_meta ) {\n doc.x_meta.collection_items.forEach( function(id) {\n emit(doc.content.name, id);\n });\n }\n}","reduce":"_count"}},"language":"javascript"}