8 use Data::Dump qw(dump);
13 use Digest::MD5 qw(md5_hex);
15 my $UserID = $ENV{UserID} || die "usage: UserID=1234 key=abcd $0";
16 my $key = $ENV{key} || die "key required";
18 my $FETCH = $ENV{FETCH} || 0;
20 my $db = CouchDB->new('10.60.0.92', 5984);
21 eval { $db->put("zotero_$UserID") }; # create user database
25 warn "# create $view";
26 $db->put("zotero_$UserID/_design/zotero" => decode_json $view)
29 eval { $db->put("rt") }; # create RT database
31 my @urls = map { "https://api.zotero.org/users/$UserID/$_?format=atom&content=json&order=dateModified&sort=desc" } qw( collections items );
32 # we don't need to fetch tags since we can generate using CouchDB views
34 my $url = shift @urls;
42 $url .= '&key=' . $key;
44 my $file = $UserID . '.' . md5_hex($url) . '.atom';
45 $FETCH = 1 if ! -e $file;
47 warn "# $url -> $file\n";
48 if ( $FETCH && mirror( $url => $file ) == RC_NOT_MODIFIED ) {
53 my $feed = XMLin( $file );
54 #warn "# feed ",dump($feed);
58 $link =~ s{.+/(items|collections)/}{}; # leave just ID
63 foreach my $entry ( keys %{ $feed->{entry} } ) {
64 warn "# entry $entry ",dump($entry);
65 my $id = link_to_id $entry;
67 my $item = $feed->{entry}->{$entry};
68 warn "# entry $entry ",dump($item);
70 foreach my $i ( 0 .. $#{ $item->{link} } ) {
71 my $link = $item->{link}->[$i];
72 warn "# link $id $i:",dump($link);
74 $item->{link}->[$i]->{key} = link_to_id $link->{href};
76 if ( $link->{rel} eq 'up' ) {
77 push @{ $tree->{$key} }, $id;
81 if ( exists $item->{content} ) {
82 my $type = ( grep { exists $item->{content}->{$_} } qw(zapi:type type) )[0];
83 warn "# content has $type";
85 $item->{zapi}->{etag} = $item->{content}->{'zapi:etag'} if exists $item->{content}->{'zapi:etag'};
87 $type = $item->{zapi}->{type} = $item->{content}->{$type};
89 if ( $type =~ m/json/ ) {
91 my $json = $item->{content}->{content};
93 $json = $item->{content} = decode_json $json;
94 warn "# json $id ", dump $json;
96 foreach my $tag ( @{ $json->{tags} } ) {
98 warn "# tag $id $tag\n";
99 next unless $tag =~ m/#(\d+)/; # XXX RT number in tag
100 push @{ $ticket_items->{$1} }, $id;
104 warn "ERROR: $type not decoded!";
108 foreach my $zapi ( grep { m/^zapi:/ } keys %$item ) {
111 $item->{zapi}->{$name} = delete $item->{$zapi};
114 $items->{$id} = $item;
116 $db->modify( "zotero_$UserID/$id" => $item );
120 delete $feed->{entry};
121 warn "# feed without entry ",dump( $feed );
123 if ( my @next = map { $_->{href} } grep { $_->{rel} eq 'next' && $_->{type} eq 'application/atom+xml' } @{ $feed->{link} } ) {
124 warn "## next ",dump(@next);
129 if ( $url = shift @urls ) {
130 warn "## next url $url";
134 warn "# tree ",dump( $tree );
136 warn "# ticket_items ",dump( $ticket_items );
139 my $rt = RT::Client::REST->new(
140 server => 'http://rt.rot13.org/rt',
144 $rt->login(username => $ENV{RT_USER}, password => $ENV{RT_PASSWORD});
146 foreach my $nr ( keys %$ticket_items ) {
148 my $ticket = eval { $rt->show(type => 'ticket', id => $nr) };
149 warn "# ticket $nr ",dump($ticket);
153 $ticket->{zotero_items} = $ticket_items->{$nr};
155 my $modified = $db->modify( "rt/$nr" => sub {
157 $doc->{$_} = $ticket->{$_} foreach keys %$ticket;
161 warn "# modified ",dump($modified);
163 # copy attachments to CouchDB (they never change, so do it just once
164 if ( my @attachment_ids = $rt->get_attachment_ids( id => $nr ) ) {
166 warn "# get_attachment_ids = ",dump( @attachment_ids );
167 my $doc = $db->get("rt/$nr");
170 foreach my $attachment_id ( @attachment_ids ) {
171 my $attachment = $rt->get_attachment( parent_id => $nr, id => $attachment_id );
172 if ( $attachment->{Filename} && $attachment->{ContentEncoding} eq 'base64' ) {
173 #$attachment->{Filename} ||= $attachment_id;
174 my $content = delete $attachment->{Content};
175 if ( ! exists $doc->{_attachments}->{ $attachment->{Filename} } ) {
176 utf8::encode($content) || warn "utf8::encode error!";
177 warn "# extracted ",length( $content ), " bytes";
178 warn "## attachment ",dump( $attachment );
179 my $url = sprintf 'rt/%d/%s?rev=%s', $nr, uri_escape($attachment->{Filename}), $modified->{rev};
180 # $modified = $db->request( PUT => $url, $content, $attachment->{ContentType} );
183 push @attachments, $attachment;
187 $db->modify( "rt/$nr" => sub {
189 $doc->{attachments} = [ @attachments ];
190 warn "## attachments on $nr = ", $#attachments + 1;
196 if ( $ticket->{Queue} !~ m/ILL/i ) {
197 warn "SKIP $ticket not in ILL queue!";
201 foreach my $id ( @{ $ticket_items->{$nr} } ) {
202 warn "# item $id ",dump( $items->{$id} );
204 # $rt->comment( ticket_id => $nr, message => dump( $items->{$id} ) );
211 {"_id":"_design/zotero","language":"javascript","views":{"itemType":{"map":"function(doc) {\n emit(doc.zapi.itemType,1);\n}","reduce":"_count"},"updated":{"map":"function(doc) {\n emit(doc.updated,1);\n}","reduce":"_count"},"tags":{"map":"function(doc) {\n \n doc.content.tags.forEach( function(v) {\n emit(v, doc._id);\n });\n}","reduce":"_count"},"link_up":{"map":"function(doc) {\n if ( doc.link[1].rel == 'up' )\n emit( doc.link[1].key, doc._id );\n}","reduce":"_count"},"year,publisher":{"map":"function(doc) {\n if ( doc.zapi.year )\n emit([doc.zapi.year, doc.content.publisher], 1);\n}","reduce":"_count"}}}