output missing year in biblio
[koha-bibliografija] / html.pl
1 #!/usr/bin/perl
2
3 # LC_COLLATE=hr_HR.utf8 KOHA_CONF=/etc/koha/sites/ffzg/koha-conf.xml ./html.pl
4
5 use warnings;
6 use strict;
7
8 use DBI;
9 use Data::Dump qw(dump);
10 use autodie;
11 use locale;
12 use Text::Unaccent;
13 use Carp qw(confess);
14 use utf8;
15
16 use lib '/srv/koha_ffzg';
17 use C4::Context;
18 use XML::LibXML;
19 use XML::LibXSLT;
20
21 my $dbh = C4::Context->dbh;
22
23 sub debug {
24         my ($title, $data) = @_;
25         print "# $title ",dump($data), $/ if $ENV{DEBUG};
26 }
27
28 my $xslfilename = 'compact.xsl';
29
30 my $azvo_group_title = {
31 'znanstveno nastavni' => qr/(profesor|docent|znanstveni)/i,
32 'lektori i predavači' => qr/(lektor|predavač)/i,
33 'asistenti i novaci' => qr/(asistent|novak)/i,
34 };
35
36 my $auth_header;
37 my $auth_department;
38 my $auth_group;
39 my @authors;
40 my $department_in_sum;
41
42 my $skip;
43
44 my $sth_auth = $dbh->prepare(q{
45 select
46         authid,
47         ExtractValue(marcxml,'//datafield[@tag="100"]/subfield[@code="a"]') as full_name,
48         ExtractValue(marcxml,'//datafield[@tag="680"]/subfield[@code="a"]') as department,
49         ExtractValue(marcxml,'//datafield[@tag="680"]/subfield[@code="i"]') as academic_title
50 from auth_header
51 });
52
53 $sth_auth->execute();
54 while( my $row = $sth_auth->fetchrow_hashref ) {
55         if ( $row->{department} !~ m/Filozofski fakultet u Zagrebu/ ) {
56                 push @{ $skip->{nije_ffzg} }, $row;
57                 next;
58         }
59         $auth_header->{ $row->{authid} } = $row->{full_name};
60         $row->{department} =~ s/, Filozofski fakultet u Zagrebu.*$//;
61         $row->{department} =~ s/^.+\.\s*//;
62         $row->{department} =~ s/\s+$//s;
63         my $group;
64         foreach my $title ( keys %$azvo_group_title ) {
65                 if ( $row->{academic_title} =~ $azvo_group_title->{$title} ) {
66                         $group = $title;
67                         last;
68                 }
69         }
70         if ( $group ) {
71                 $row->{academic_group} = $group;
72                 $auth_group->{ $row->{authid} } = $group;
73                 $skip->{group_stat}->{$group}++;
74         } else {
75                 push @{ $skip->{no_academic_group} }, $row;
76         }
77
78 #       warn "# ", dump( $row );
79         push @{ $auth_department->{ $row->{department} } }, $row->{authid};
80         push @authors, $row;
81         $department_in_sum->{ $row->{department} }++;
82 }
83
84 foreach my $department ( keys %$department_in_sum ) {
85         $department_in_sum->{$department} = 0 unless $department =~ m/(centar|croaticum|katedra|odsjek)/i;
86 }
87
88 debug 'auth_department' => $auth_department;
89 debug 'auth_group' => $auth_group;
90 debug 'department_in_sum' => $department_in_sum;
91
92
93 my $authors;
94 my $marcxml;
95
96 my $sth_select_authors  = $dbh->prepare(q{
97 select
98         biblionumber,
99         itemtype,
100         marcxml
101 from biblioitems
102 where
103         agerestriction > 0
104 });
105
106 =for sql
107 --      ExtractValue(marcxml,'//datafield[@tag="100"]/subfield[@code="9"]') as first_author,
108 --      ExtractValue(marcxml,'//datafield[@tag="700"]/subfield[@code="9"]') as other_authors,
109 --      ExtractValue(marcxml,'//datafield[@tag="942"]/subfield[@code="t"]') as category,
110
111 --      and SUBSTR(ExtractValue(marcxml,'//controlfield[@tag="008"]'),8,4) between 2008 and 2013
112 -- order by SUBSTR(ExtractValue(marcxml,'//controlfield[@tag="008"]'),8,4) desc
113 =cut
114
115 my $biblio_year;
116 my $type_stats;
117
118 my $parser = XML::LibXML->new();
119 $parser->recover_silently(0); # don't die when you find &, >, etc
120 my $style_doc = $parser->parse_file($xslfilename);
121 my $xslt = XML::LibXSLT->new();
122 my $parsed = $xslt->parse_stylesheet($style_doc);
123
124 my $biblio_html;
125 my $biblio_parsed;
126 my $biblio_data;
127
128 open(my $xml_fh, '>', '/tmp/bibliografija.xml') if $ENV{XML};
129
130 sub biblioitem_html {
131         my ($biblionumber, $parse_only) = @_;
132
133         return $biblio_html->{$biblionumber} if exists $biblio_html->{$biblionumber} && ! $parse_only;
134
135         my $xmlrecord = $marcxml->{$biblionumber} || confess "missing $biblionumber marcxml";
136
137         print $xml_fh $xmlrecord if $ENV{XML};
138
139         my $source = eval { $parser->parse_string($xmlrecord) };
140         if ( $@ ) {
141 #               warn "SKIP $biblionumber corrupt XML";
142                 push @{ $skip->{XML_corrupt} }, $biblionumber;
143                 return;
144         }
145
146         if ( $parse_only ) {
147                 $biblio_parsed->{$biblionumber} = $source;
148                 return $source;
149         }
150
151         my $transformed = $parsed->transform($source);
152         $biblio_html->{$biblionumber} = $parsed->output_string( $transformed );
153
154         delete $biblio_parsed->{$biblionumber};
155
156         return $biblio_html->{$biblionumber};
157 }
158
159 $sth_select_authors->execute();
160 while( my $row = $sth_select_authors->fetchrow_hashref ) {
161 #       warn dump($row),$/;
162
163         my $biblio;
164
165         $marcxml->{ $row->{biblionumber} } = $row->{marcxml};
166
167         my $doc = biblioitem_html( $row->{biblionumber}, 1 );
168         if ( ! $doc ) {
169 #               warn "ERROR can't parse MARCXML ", $row->{biblionumber}, " ", $row->{marcxml}, "\n";
170                 next;
171         }
172
173         my $root = $doc->documentElement;
174 =for leader
175         my @leaders = $root->getElementsByLocalName('leader');
176         if (@leaders) {
177                 my $leader = $leaders[0]->textContent;
178                 warn "leader $leader\n";
179         }
180 =cut
181
182         my $extract = {
183                 '008' => undef,
184                 '100' => '9',
185                 '680' => 'i',
186                 '700' => '(9|4)',
187                 '942' => '(t|r|v)'
188         };
189
190         my $data;
191
192         foreach my $elt ($root->getChildrenByLocalName('*')) {
193                 my $tag = $elt->getAttribute('tag');
194                 next if ! $tag;
195                 next unless exists $extract->{ $tag };
196
197         if ($elt->localname eq 'controlfield') {
198                         if ( $tag eq '008' ) {
199                                 my $content = $elt->textContent;
200                                 my $year = substr($content, 7, 4 );
201                                 if ( $year !~ m/^\d+$/ ) {
202                                         $year = 0;
203                                         push @{ $skip->{invalid_year} }, $row->{biblionumber};
204                                 }
205                                 $biblio_year->{ $row->{biblionumber} } = $data->{year} = $year;
206                                 $data->{'008'} = $content;
207                         }
208                         next;
209         } elsif ($elt->localname eq 'datafield') {
210                         my $sf_data;
211             foreach my $sfelt ($elt->getChildrenByLocalName('subfield')) {
212                 my $sf = $sfelt->getAttribute('code');
213                                 next unless $sf =~ m/$extract->{$tag}/;
214                                 if ( exists $sf_data->{$sf} ) {
215                                         $sf_data->{$sf} .= " " . $sfelt->textContent();
216                                 } else {
217                                         $sf_data->{$sf} = $sfelt->textContent();
218                                 }
219                         }
220                         push @{ $data->{$tag} }, $sf_data if $sf_data;
221                 }
222         }
223
224         if ( ! defined $data->{year} ) {
225                 warn "MISSING year in ", $row->{biblionumber};
226         } elsif ( $data->{year} < 2008 ) {
227                 push @{ $skip->{year_lt_2008} }, $row->{biblionumber};
228                 next;
229         } elsif ( $data->{year} > 2013 ) {
230                 push @{ $skip->{year_gt_2013} }, $row->{biblionumber};
231                 next;
232         }
233
234 #       warn "# ", $row->{biblionumber}, " data ",dump($data);
235
236         my $category = $data->{942}->[0]->{'t'};
237         if ( ! $category ) {
238 #               warn "# SKIP ", $row->{biblionumber}, " no category in ", dump($data);
239                 push @{ $skip->{no_category} }, $row->{biblionumber};
240                 next;
241         }
242
243         my $have_100 = 1;
244
245         if ( exists $data->{100} ) {
246                         my @first_author = map { $_->{'9'} } @{ $data->{100} };
247                         foreach my $authid ( @first_author ) {
248                                 push @{ $authors->{$authid}->{aut}->{ $category } }, $row->{biblionumber};
249                         }
250         } else {
251                 $have_100 = 0;
252         }
253
254         my $have_edt;
255
256         if ( exists $data->{700} ) {
257                         foreach my $auth ( @{ $data->{700} } ) {
258                                 my $authid = $auth->{9} || next;
259                                 my $type   = $auth->{4} || next; #die "no 4 in ",dump($data);
260
261                                 $type_stats->{$type}++;
262
263                                 if ( $type =~ m/(edt|trl|com|ctb)/ ) {
264                                         push @{ $authors->{$authid}->{sec}->{ $category } }, $row->{biblionumber};
265                                         push @{ $authors->{$authid}->{$1}->{ $category } }, $row->{biblionumber};
266                                 } elsif ( $type =~ m/aut/ ) {
267                                         if ( ! $have_100 ) {
268                                                 $have_edt = grep { exists $_->{4} && $_->{4} =~ m/edt/ } @{ $data->{700} } if ! defined $have_edt;
269                                                 if ( $have_edt ) {
270                                                         $skip->{ have_700_edt }->{ $row->{biblionumber} }++;
271                                                 } else {
272                                                         push @{ $authors->{$authid}->{aut}->{ $category } }, $row->{biblionumber};
273                                                 }
274                                         } else {
275                                                 push @{ $authors->{$authid}->{aut}->{ $category } }, $row->{biblionumber};
276                                         }
277                                 } else {
278 #                                       warn "# SKIP ", $row->{biblionumber}, ' no 700$4 in ', dump($data);
279                                         $skip->{ 'no_700$4' }->{ $row->{biblionumber} }++;
280                                 }
281                         }
282                         delete $data->{700};
283         }
284
285         $biblio_data->{ $row->{biblionumber} } = $data;
286
287 }
288
289 debug 'authors' => $authors;
290 debug 'type_stats' => $type_stats;
291 debug 'skip' => $skip;
292 debug 'biblio_year' => $biblio_year;
293 debug 'biblio_data' => $biblio_data;
294
295 my $category_label;
296 my $sth_categories = $dbh->prepare(q{
297 select authorised_value, lib from authorised_values where category = 'BIBCAT'
298 });
299 $sth_categories->execute();
300 while( my $row = $sth_categories->fetchrow_hashref ) {
301         $category_label->{ $row->{authorised_value} } = $row->{lib};
302
303 }
304 debug 'category_label' => $category_label;
305
306 sub html_title {
307         return qq|<html>
308 <head>
309 <meta charset="UTF-8">
310 <title>|, join(" ", @_), qq|</title>
311 <link href="style.css" type="text/css" rel="stylesheet" />
312 </head>
313 <body>
314 |;
315 }
316
317 sub html_end {
318         return qq|</body>\n</html>\n|;
319 }
320
321 mkdir 'html' unless -d 'html';
322
323 open(my $index, '>:encoding(utf-8)', 'html/index.new');
324 print $index html_title('Bibliografija Filozofskog fakulteta');
325
326 my $first_letter = '';
327
328 debug 'authors' => \@authors;
329
330 sub li_biblio {
331         my ($biblionumber) = @_;
332         return qq|<li>|,
333                 qq|<a href="https://koha.ffzg.hr/cgi-bin/koha/opac-detail.pl?biblionumber=$biblionumber">$biblionumber</a>|,
334                 biblioitem_html($biblionumber),
335                 qq|<a href="https://koha.ffzg.hr:8443/cgi-bin/koha/cataloguing/addbiblio.pl?biblionumber=$biblionumber">edit</a>|,
336                 qq|</li>\n|;
337 }
338
339 sub author_html {
340         my ( $fh, $authid, $type, $label ) = @_;
341
342         return unless exists $authors->{$authid}->{$type};
343
344         print $fh qq|<h2>$label</h2>\n|;
345
346         foreach my $category ( sort keys %{ $authors->{$authid}->{$type} } ) {
347                 my $label = $category_label->{$category} || 'Bez kategorije';
348                 print $fh qq|<h3>$label</h3>\n<ul>\n|;
349                 foreach my $biblionumber ( @{ $authors->{$authid}->{$type}->{$category} } ) {
350                         print $fh li_biblio( $biblionumber );
351                 }
352                 print $fh qq|</ul>\n|;
353         }
354 }
355
356 foreach my $row ( sort { $a->{full_name} cmp $b->{full_name} } @authors ) {
357
358         my $first = substr( $row->{full_name}, 0, 1 );
359         if ( $first ne $first_letter ) {
360                 print $index qq{</ul>\n} if $first_letter;
361                 $first_letter = $first;
362                 print $index qq{<h1>$first</h1>\n<ul>\n};
363         }
364         print $index qq{<li><a href="}, $row->{authid}, qq{.html">}, $row->{full_name}, "</a></li>\n";
365
366         my $path = "html/$row->{authid}";
367         open(my $fh, '>:encoding(utf-8)', "$path.new");
368         print $fh html_title($row->{full_name}, "bibliografija");
369         print $fh qq|<h1>$row->{full_name} - bibliografija za razdoblje 2008-2013</h1>|;
370
371         author_html( $fh, $row->{authid}, 'aut' => 'Primarno autorstvo' );
372         author_html( $fh, $row->{authid}, 'sec' => 'Uredništva, prijevodi, krička izdanja' );
373
374         print $fh html_end;
375         close($fh);
376         rename "$path.new", "$path.html";
377
378 }
379
380 print $index html_end;
381 close($index);
382 rename 'html/index.new', 'html/index.html';
383
384 debug 'auth_header' => $auth_header;
385
386
387 my $department_category_author;
388 foreach my $department ( sort keys %$auth_department ) {
389         foreach my $authid ( sort @{ $auth_department->{$department} } ) {
390                 my   @categories = keys %{ $authors->{$authid}->{aut} };
391                 push @categories,  keys %{ $authors->{$authid}->{sec} };
392                 foreach my $category ( sort @categories ) {
393                         push @{ $department_category_author->{$department}->{$category} }, $authid;
394                         push @{ $department_category_author->{''}->{$category} }, $authid if $department_in_sum->{$department};
395                 }
396         }
397 }
398
399 debug 'department_category_author' => $department_category_author;
400
401 mkdir 'html/departments' unless -d 'html/departments';
402
403 sub unique_biblionumber {
404         my @v = @_;
405         my $u;
406         $u->{$_}++ foreach @v;
407         return sort { $biblio_year->{$b} <=> $biblio_year->{$a} || $a <=> $b } keys %$u;
408 }
409
410 open(my $dep_fh, '>:encoding(utf-8)', 'html/departments/index.new');
411 print $dep_fh html_title('Odsijeci Filozofskog fakulteta u Zagrebu'), qq|<ul>\n|;
412 foreach my $department ( sort keys %$department_category_author ) {
413         my $dep = $department || 'Nema odsjeka';
414         my $dep_file = unac_string('utf-8',$dep);
415         print $dep_fh qq|<li><a href="$dep_file.html">$dep</a></li>\n|;
416         open(my $fh, '>:encoding(utf-8)', "html/departments/$dep_file.new");
417
418         print $fh html_title($department . ' bibliografija');
419         print $fh qq|<h1>$department bibliografija</h1>\n|;
420
421         print $fh qq|<h2>Primarno autorstvo</h2>\n|;
422
423         foreach my $category ( sort keys %{ $department_category_author->{$department} } ) {
424
425                 my @authids = @{ $department_category_author->{$department}->{$category} };
426                 next unless @authids;
427
428                 my @biblionumber = unique_biblionumber map { @{ $authors->{$_}->{aut}->{$category} } } grep { exists $authors->{$_}->{aut}->{$category} } @authids;
429                 my $unique;
430                 $unique->{$_}++ foreach @biblionumber;
431
432                 next unless @biblionumber;
433
434                 my $label = $category_label->{$category} || 'Bez kategorije';
435                 print $fh qq|<h3>$label</h3>\n<ul>\n|;
436
437                 print $fh li_biblio( $_ ) foreach @biblionumber;
438
439                 print $fh qq|</ul>|;
440         }
441
442
443         print $fh qq|<h2>Sekundarno autorstvo</h2>\n|;
444
445         foreach my $category ( sort keys %{ $department_category_author->{$department} } ) {
446
447                 my @authids = @{ $department_category_author->{$department}->{$category} };
448                 next unless @authids;
449
450                 my @biblionumber = unique_biblionumber map { @{ $authors->{$_}->{sec}->{$category} } } grep { exists $authors->{$_}->{sec}->{$category} } @authids;
451
452                 next unless @biblionumber;
453
454                 my $label = $category_label->{$category} || 'Bez kategorije';
455                 print $fh qq|<h3>$label</h3>\n<ul>\n|;
456
457                 print $fh li_biblio( $_ ) foreach @biblionumber;
458
459                 print $fh qq|</ul>|;
460         }
461
462
463         print $fh html_end;
464         close($fh);
465         rename "html/departments/$dep_file.new", "html/departments/$dep_file.html";
466 }
467 print $dep_fh qq|</ul>\n|, html_end;
468 close($dep_fh);
469 rename 'html/departments/index.new', 'html/departments/index.html';
470
471 my $azvo_stat_biblio;
472
473 foreach my $department ( sort keys %$department_category_author ) {
474         foreach my $category ( sort keys %{ $department_category_author->{$department} } ) {
475                 foreach my $authid ( @{ $department_category_author->{$department}->{$category} } ) {
476                         my $group = $auth_group->{$authid};
477                         if ( ! $group ) {
478                                 push @{ $skip->{no_auth_group} }, $authid;
479                                 next;
480                         }
481                         foreach my $type ( keys %{ $authors->{$authid} } ) {
482                                 next unless exists $authors->{$authid}->{$type}->{$category};
483                                 push @{ $azvo_stat_biblio->{ $department }->{ $category }->{ $type }->{$group} },  @{ $authors->{$authid}->{$type}->{$category} };
484                                 push @{ $azvo_stat_biblio->{ $department }->{ $category }->{ $type }->{''} },  @{ $authors->{$authid}->{$type}->{$category} };
485                         }
486                 }
487                 foreach my $type ( keys %{ $azvo_stat_biblio->{ $department }->{ $category } } ) {
488                         foreach my $group ( keys %{ $azvo_stat_biblio->{ $department }->{ $category }->{ $type } } ) {
489                                 my @biblios = unique_biblionumber @{ $azvo_stat_biblio->{ $department }->{ $category }->{ $type }->{ $group } };
490                                 $azvo_stat_biblio->{ $department }->{ $category }->{ $type }->{ $group } = [ @biblios ];
491                         }
492                 }
493         }
494 }
495
496 debug 'azvo_stat_biblio' => $azvo_stat_biblio;
497
498 my @report_lines;
499 my @report_labels;
500
501 my $label;
502 my $sub_labels;
503 open(my $report, '<:encoding(utf-8)', 'AZVO.txt');
504 while( <$report> ) {
505         chomp;
506         if ( /^([^\t]+)\t+(.+)/ ) {
507                 $label = $1;
508                 push @report_labels, $label;
509                 my $type = [ map { m/\s+/ ? [ split(/\s+/,$_) ] : [ $_, 'aut' ] } split (/\s*\+\s*/, $2) ];
510                 push @report_lines, [ $label, @$type ];
511         } elsif ( /^\t+([^\t]+):\t+(\d+)(\w*)\t*(.*)$/ ) {
512                 push @{ $sub_labels->{$label} }, [ $1, $2, $3, $4 ];
513                 my $sub_label = $1;
514                 pop (@report_labels) if ( $report_labels[ $#report_labels ] =~ m/^$label$/ ); # remove partial name
515                 push @report_labels, $label . $sub_label;
516         } else {
517                 die "ERROR: [$_]\n";
518         }
519 }
520
521 debug 'report_lines', \@report_lines;
522 debug 'sub_labels', $sub_labels;
523 debug 'report_labels', \@report_labels;
524
525 my @departments = ( sort { lc($a) cmp lc($b) } keys %$azvo_stat_biblio );
526
527 debug 'departments' => \@departments;
528
529 my $department2col;
530 $department2col->{ $departments[$_] } = $_ foreach ( 0 .. $#departments );
531 my $label2row;
532 $label2row->{ $report_labels[$_] } = $_ foreach ( 0 .. $#report_labels );
533
534 my $table;
535
536 sub table_count {
537         my $label = shift @_;
538         my $department = shift @_;
539         my $group = shift @_;
540         my @biblionumbers = @_;
541         my $unique;
542         $unique->{$_}++ foreach @biblionumbers;
543         $table->{$group}->[ $label2row->{ $label } ]->[ $department2col->{$department} ] = scalar keys %$unique;
544 }
545
546 foreach my $group ( '', keys %$azvo_group_title ) {
547
548 foreach my $department ( @departments ) {
549         foreach my $line ( @report_lines ) {
550                 my $label = $line->[0];
551                 my @biblionumbers;
552                 foreach ( 1 .. $#$line ) {
553                         my ( $category, $type ) = @{ $line->[ $_ ] };
554                         my $b = $azvo_stat_biblio->{ $department }->{$category}->{$type}->{$group};
555                         push @biblionumbers, @$b if $b;
556                 }
557                 if ( $sub_labels->{$label} ) {
558                         my $sub_stats;
559                         foreach my $biblionumber ( @biblionumbers ) {
560                                 my $data = $biblio_data->{$biblionumber} || die "can't find biblionumber $biblionumber";
561                                 foreach my $sub_label ( @{ $sub_labels->{$label} } ) {
562                                         my ( $sub_label, $field, $sf, $regex ) = @$sub_label;
563                                         if ( ! $regex ) {
564                                                 push @{ $sub_stats->{ $sub_label } }, $biblionumber;
565                                                 last;
566                                         }
567                                         if ( $field < 100 ) {
568                                                 if ( $data->{$field} =~ m/$regex/ ) {
569                                                         push @{ $sub_stats->{ $sub_label } }, $biblionumber;
570                                                         last;
571                                                 }
572                                         } else {
573                                                 if ( exists $data->{$field}->[0]->{$sf} && $data->{$field}->[0]->{$sf} =~ m/$regex/ ) {
574                                                         push @{ $sub_stats->{ $sub_label } }, $biblionumber;
575                                                         last;
576                                                 }
577                                         }
578                                 }
579                         }
580                         foreach my $sub_label ( keys %$sub_stats ) {
581                                 my $full_label = $label . $sub_label;
582                                 table_count $full_label, $department, $group, @{ $sub_stats->{$sub_label} };
583                         }
584                 } else {
585                         table_count $label, $department, $group, @biblionumbers;
586                 }
587         }
588 }
589
590 } # group
591
592 debug 'table', $table;
593
594 open(my $fh, '>:encoding(utf-8)', 'html/azvo.new');
595
596 print $fh html_title('AZVO tablica');
597
598 foreach my $group ( keys %$table ) {
599
600                 print $fh "<h1>$group</h1>" if $group;
601
602                 print $fh "<table border=1>\n";
603                 print $fh "<tr><th></th>";
604                 print $fh "<th>$_</th>" foreach @departments;
605                 print $fh "</tr>\n";
606
607                 foreach my $row ( 0 .. $#{ $table->{$group} } ) {
608                         print $fh "<tr><th>", $report_labels[$row], "</th>\n";
609                         print $fh " <td>", $table->{$group}->[ $row ]->[ $_ ] || '', "</td>\n" foreach 0 .. $#departments;
610                         print $fh "</tr>\n";
611                 }
612
613                 print $fh "</table>\n";
614
615 } # group
616
617 print $fh html_end;
618 close($fh);
619 rename 'html/azvo.new', 'html/azvo.html';
620