X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=C4%2FImportBatch.pm;h=02f242433dca56e4323411fa4d825a15a3b685ec;hb=622430cfb5bb1b9886e00a835d96b8434d6ef1db;hp=5149f9bff25d66dff5de5dc23baa24b0545d3066;hpb=6e71b80ca39e6f1d2c31659a9a656de807ef3f32;p=koha.git diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index 5149f9bff2..02f242433d 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -26,6 +26,7 @@ use C4::Biblio; use C4::Items; use C4::Charset; use C4::AuthoritiesMarc; +use Koha::MarcModificationTemplates; use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); @@ -205,7 +206,7 @@ sub AddImportBatch { my (@fields, @vals); foreach (qw( matcher_id template_id branchcode overlay_action nomatch_action item_action - import_status batch_type file_name comments )) { + import_status batch_type file_name comments record_type )) { if (exists $params->{$_}) { push @fields, $_; push @vals, $params->{$_}; @@ -255,7 +256,7 @@ sub AddBiblioToBatch { my $z3950random = shift; my $update_counts = @_ ? shift : 1; - my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, $z3950random); + my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, $z3950random, C4::Context->preference('marcflavour')); _add_biblio_fields($import_record_id, $marc_record); _update_batch_record_counts($batch_id) if $update_counts; return $import_record_id; @@ -270,7 +271,7 @@ sub AddBiblioToBatch { sub ModBiblioInBatch { my ($import_record_id, $marc_record) = @_; - _update_import_record_marc($import_record_id, $marc_record); + _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour')); _update_biblio_fields($import_record_id, $marc_record); } @@ -278,7 +279,7 @@ sub ModBiblioInBatch { =head2 AddAuthToBatch my $import_record_id = AddAuthToBatch($batch_id, $record_sequence, - $marc_record, $encoding, $z3950random, $update_counts); + $marc_record, $encoding, $z3950random, $update_counts, [$marc_type]); =cut @@ -289,8 +290,11 @@ sub AddAuthToBatch { my $encoding = shift; my $z3950random = shift; my $update_counts = @_ ? shift : 1; + my $marc_type = shift || C4::Context->preference('marcflavour'); + + $marc_type = 'UNIMARCAUTH' if $marc_type eq 'UNIMARC'; - my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random); + my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random, $marc_type); _add_auth_fields($import_record_id, $marc_record); _update_batch_record_counts($batch_id) if $update_counts; return $import_record_id; @@ -305,14 +309,15 @@ sub AddAuthToBatch { sub ModAuthInBatch { my ($import_record_id, $marc_record) = @_; - _update_import_record_marc($import_record_id, $marc_record); + my $marcflavour = C4::Context->preference('marcflavour'); + _update_import_record_marc($import_record_id, $marc_record, $marcflavour eq 'UNIMARC' ? 'UNIMARCAUTH' : 'USMARC'); } =head2 BatchStageMarcRecords ($batch_id, $num_records, $num_items, @invalid_records) = - BatchStageMarcRecords($record_type, $encoding, $marc_records, $file_name, + BatchStageMarcRecords($encoding, $marc_records, $file_name, $marc_modification_template, $comments, $branch_code, $parse_items, $leave_as_staging, $progress_interval, $progress_callback); @@ -324,11 +329,12 @@ sub BatchStageMarcRecords { my $encoding = shift; my $marc_records = shift; my $file_name = shift; + my $marc_modification_template = shift; my $comments = shift; my $branch_code = shift; my $parse_items = shift; my $leave_as_staging = shift; - + # optional callback to monitor status # of job my $progress_interval = 0; @@ -346,6 +352,7 @@ sub BatchStageMarcRecords { batch_type => 'batch', file_name => $file_name, comments => $comments, + record_type => $record_type, } ); if ($parse_items) { SetImportBatchItemAction($batch_id, 'always_add'); @@ -353,6 +360,8 @@ sub BatchStageMarcRecords { SetImportBatchItemAction($batch_id, 'ignore'); } + my $marc_type = C4::Context->preference('marcflavour'); + $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth'); my @invalid_records = (); my $num_valid = 0; my $num_items = 0; @@ -367,14 +376,20 @@ sub BatchStageMarcRecords { &$progress_callback($rec_num); } my ($marc_record, $charset_guessed, $char_errors) = - MarcToUTF8Record($marc_blob, C4::Context->preference("marcflavour"), $encoding); + MarcToUTF8Record($marc_blob, $marc_type, $encoding); $encoding = $charset_guessed unless $encoding; + ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template ); + my $import_record_id; if (scalar($marc_record->fields()) == 0) { push @invalid_records, $marc_blob; } else { + + # Normalize the record so it doesn't have separated diacritics + SetUTF8Flag($marc_record); + $num_valid++; if ($record_type eq 'biblio') { $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); @@ -383,7 +398,7 @@ sub BatchStageMarcRecords { $num_items += scalar(@import_items_ids); } } elsif ($record_type eq 'auth') { - $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); + $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0, $marc_type); } } } @@ -429,7 +444,7 @@ sub AddItemsToImportBiblio { if ($#import_items_ids > -1) { _update_batch_record_counts($batch_id) if $update_counts; - _update_import_record_marc($import_record_id, $marc_record); + _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour')); } return @import_items_ids; } @@ -503,7 +518,7 @@ sub BatchFindDuplicates { =head2 BatchCommitRecords - my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = + my ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored) = BatchCommitRecords($batch_id, $framework, $progress_interval, $progress_callback); @@ -528,6 +543,7 @@ sub BatchCommitRecords { my $num_added = 0; my $num_updated = 0; my $num_items_added = 0; + my $num_items_replaced = 0; my $num_items_errored = 0; my $num_ignored = 0; # commit (i.e., save, all records in the batch) @@ -544,6 +560,7 @@ sub BatchCommitRecords { LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); + my $marcflavour = C4::Context->preference('marcflavour'); my $rec_num = 0; while (my $rowref = $sth->fetchrow_hashref) { $record_type = $rowref->{'record_type'}; @@ -556,6 +573,14 @@ sub BatchCommitRecords { next; } + my $marc_type; + if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') { + $marc_type = 'UNIMARCAUTH'; + } elsif ($marcflavour eq 'UNIMARC') { + $marc_type = 'UNIMARC'; + } else { + $marc_type = 'USMARC'; + } my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'}); if ($record_type eq 'biblio') { @@ -578,13 +603,14 @@ sub BatchCommitRecords { my $biblioitemnumber; ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework); $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; - if ($item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); + if ($item_result eq 'create_new' || $item_result eq 'replace') { + my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); $num_items_added += $bib_items_added; + $num_items_replaced += $bib_items_replaced; $num_items_errored += $bib_items_errored; } } else { - my $authid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record)); + $recordid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record)); $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?"; } my $sth = $dbh->prepare_cached($query); @@ -601,22 +627,24 @@ sub BatchCommitRecords { # remove item fields so that they don't get # added again if record is reverted - my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}); + # FIXME: GetXmlBiblio output should not contain item info any more! So the next foreach should not be needed. Does not hurt either; may remove old 952s that should not have been there anymore. + my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}, $marc_type); foreach my $item_field ($old_marc->field($item_tag)) { $old_marc->delete_field($item_field); } - $oldxml = $old_marc->as_xml(); + $oldxml = $old_marc->as_xml($marc_type); ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'}); $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; - if ($item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); + if ($item_result eq 'create_new' || $item_result eq 'replace') { + my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); $num_items_added += $bib_items_added; + $num_items_replaced += $bib_items_replaced; $num_items_errored += $bib_items_errored; } } else { - my $oldxml = GetAuthorityXML($recordid); + $oldxml = GetAuthorityXML($recordid); ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record)); $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?"; @@ -630,10 +658,13 @@ sub BatchCommitRecords { SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); } elsif ($record_result eq 'ignore') { + $recordid = $record_match; $num_ignored++; - if ($record_type eq 'biblio' and defined $recordid and $item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); + $recordid = $record_match; + if ($record_type eq 'biblio' and defined $recordid and ( $item_result eq 'create_new' || $item_result eq 'replace' ) ) { + my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); $num_items_added += $bib_items_added; + $num_items_replaced += $bib_items_replaced; $num_items_errored += $bib_items_errored; # still need to record the matched biblionumber so that the # items can be reverted @@ -646,7 +677,7 @@ sub BatchCommitRecords { } $sth->finish(); SetImportBatchStatus($batch_id, 'imported'); - return ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored); + return ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored); } =head2 BatchCommitItems @@ -657,44 +688,73 @@ sub BatchCommitRecords { =cut sub BatchCommitItems { - my ($import_record_id, $biblionumber) = @_; + my ( $import_record_id, $biblionumber, $action ) = @_; my $dbh = C4::Context->dbh; my $num_items_added = 0; my $num_items_errored = 0; - my $sth = $dbh->prepare("SELECT import_items_id, import_items.marcxml, encoding - FROM import_items - JOIN import_records USING (import_record_id) - WHERE import_record_id = ? - ORDER BY import_items_id"); - $sth->bind_param(1, $import_record_id); + my $num_items_replaced = 0; + + my $sth = $dbh->prepare( " + SELECT import_items_id, import_items.marcxml, encoding + FROM import_items + JOIN import_records USING (import_record_id) + WHERE import_record_id = ? + ORDER BY import_items_id + " ); + $sth->bind_param( 1, $import_record_id ); $sth->execute(); - while (my $row = $sth->fetchrow_hashref()) { - my $item_marc = MARC::Record->new_from_xml(StripNonXmlChars($row->{'marcxml'}), 'UTF-8', $row->{'encoding'}); - # FIXME - duplicate barcode check needs to become part of AddItemFromMarc() - my $item = TransformMarcToKoha($dbh, $item_marc); - my $duplicate_barcode = exists($item->{'barcode'}) && GetItemnumberFromBarcode($item->{'barcode'}); - if ($duplicate_barcode) { - my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, import_error = ? WHERE import_items_id = ?"); - $updsth->bind_param(1, 'error'); - $updsth->bind_param(2, 'duplicate item barcode'); - $updsth->bind_param(3, $row->{'import_items_id'}); + + while ( my $row = $sth->fetchrow_hashref() ) { + my $item_marc = MARC::Record->new_from_xml( StripNonXmlChars( $row->{'marcxml'} ), 'UTF-8', $row->{'encoding'} ); + + # Delete date_due subfield as to not accidentally delete item checkout due dates + my ( $MARCfield, $MARCsubfield ) = GetMarcFromKohaField( 'items.onloan', GetFrameworkCode($biblionumber) ); + $item_marc->field($MARCfield)->delete_subfield( code => $MARCsubfield ); + + my $item = TransformMarcToKoha( $dbh, $item_marc ); + + my $duplicate_barcode = exists( $item->{'barcode'} ) && GetItemnumberFromBarcode( $item->{'barcode'} ); + my $duplicate_itemnumber = exists( $item->{'itemnumber'} ); + + my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ? WHERE import_items_id = ?"); + if ( $action eq "replace" && $duplicate_itemnumber ) { + # Duplicate itemnumbers have precedence, that way we can update barcodes by overlaying + ModItemFromMarc( $item_marc, $biblionumber, $item->{itemnumber} ); + $updsth->bind_param( 1, 'imported' ); + $updsth->bind_param( 2, $item->{itemnumber} ); + $updsth->bind_param( 3, $row->{'import_items_id'} ); + $updsth->execute(); + $updsth->finish(); + $num_items_replaced++; + } elsif ( $action eq "replace" && $duplicate_barcode ) { + my $itemnumber = GetItemnumberFromBarcode( $item->{'barcode'} ); + ModItemFromMarc( $item_marc, $biblionumber, $itemnumber ); + $updsth->bind_param( 1, 'imported' ); + $updsth->bind_param( 2, $item->{itemnumber} ); + $updsth->bind_param( 3, $row->{'import_items_id'} ); + $updsth->execute(); + $updsth->finish(); + $num_items_replaced++; + } elsif ($duplicate_barcode) { + $updsth->bind_param( 1, 'error' ); + $updsth->bind_param( 2, 'duplicate item barcode' ); + $updsth->bind_param( 3, $row->{'import_items_id'} ); $updsth->execute(); $num_items_errored++; } else { - my ($item_biblionumber, $biblioitemnumber, $itemnumber) = AddItemFromMarc($item_marc, $biblionumber); - my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ? WHERE import_items_id = ?"); - $updsth->bind_param(1, 'imported'); - $updsth->bind_param(2, $itemnumber); - $updsth->bind_param(3, $row->{'import_items_id'}); + my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber ); + $updsth->bind_param( 1, 'imported' ); + $updsth->bind_param( 2, $itemnumber ); + $updsth->bind_param( 3, $row->{'import_items_id'} ); $updsth->execute(); $updsth->finish(); $num_items_added++; } } - $sth->finish(); - return ($num_items_added, $num_items_errored); + + return ( $num_items_added, $num_items_replaced, $num_items_errored ); } =head2 BatchRevertRecords @@ -724,12 +784,21 @@ sub BatchRevertRecords { LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); + my $marc_type; + my $marcflavour = C4::Context->preference('marcflavour'); while (my $rowref = $sth->fetchrow_hashref) { $record_type = $rowref->{'record_type'}; if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') { $num_ignored++; next; } + if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') { + $marc_type = 'UNIMARCAUTH'; + } elsif ($marcflavour eq 'UNIMARC') { + $marc_type = 'UNIMARC'; + } else { + $marc_type = 'USMARC'; + } my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'}); @@ -749,7 +818,7 @@ sub BatchRevertRecords { } } elsif ($record_result eq 'restore') { $num_reverted++; - my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}); + my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}, $marc_type); if ($record_type eq 'biblio') { my $biblionumber = $rowref->{'matched_biblionumber'}; my ($count, $oldbiblio) = GetBiblio($biblionumber); @@ -1005,7 +1074,12 @@ sub GetBestRecordMatch { my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("SELECT candidate_match_id FROM import_record_matches - WHERE import_record_id = ? + JOIN import_records ON ( import_record_matches.import_record_id = import_records.import_record_id ) + LEFT JOIN biblio ON ( candidate_match_id = biblio.biblionumber ) + LEFT JOIN auth_header ON ( candidate_match_id = auth_header.authid ) + WHERE import_record_matches.import_record_id = ? AND + ( (import_records.record_type = 'biblio' AND biblio.biblionumber IS NOT NULL) OR + (import_records.record_type = 'auth' AND auth_header.authid IS NOT NULL) ) ORDER BY score DESC, candidate_match_id DESC"); $sth->execute($import_record_id); my ($record_id) = $sth->fetchrow_array(); @@ -1281,7 +1355,7 @@ sub GetImportRecordMatches { $sth->execute(); while (my $row = $sth->fetchrow_hashref) { if ($row->{'record_type'} eq 'auth') { - $row->{'authorized_heading'} = GetAuthorizedHeading( { authid => $row->{'candidate_match_id'} } ); + $row->{'authorized_heading'} = C4::AuthoritiesMarc::GetAuthorizedHeading( { authid => $row->{'candidate_match_id'} } ); } next if ($row->{'record_type'} eq 'biblio' && not $row->{'biblionumber'}); push @$results, $row; @@ -1320,13 +1394,13 @@ sub SetImportRecordMatches { # internal functions sub _create_import_record { - my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random) = @_; + my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random, $marc_type) = @_; my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml, record_type, encoding, z3950random) VALUES (?, ?, ?, ?, ?, ?, ?)"); - $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml(), + $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml($marc_type), $record_type, $encoding, $z3950random); my $import_record_id = $dbh->{'mysql_insertid'}; $sth->finish(); @@ -1334,12 +1408,12 @@ sub _create_import_record { } sub _update_import_record_marc { - my ($import_record_id, $marc_record) = @_; + my ($import_record_id, $marc_record, $marc_type) = @_; my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("UPDATE import_records SET marc = ?, marcxml = ? WHERE import_record_id = ?"); - $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml(C4::Context->preference('marcflavour')), $import_record_id); + $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml($marc_type), $import_record_id); $sth->finish(); } @@ -1350,9 +1424,9 @@ sub _add_auth_fields { if ($marc_record->field('001')) { $controlnumber = $marc_record->field('001')->data(); } - my $authorized_heading = GetAuthorizedHeading($marc_record); + my $authorized_heading = C4::AuthoritiesMarc::GetAuthorizedHeading({ record => $marc_record }); my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id, controlnumber, authorized_heading) VALUES (?, ?, ?)"); + my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id, control_number, authorized_heading) VALUES (?, ?, ?)"); $sth->execute($import_record_id, $controlnumber, $authorized_heading); $sth->finish(); } @@ -1403,8 +1477,7 @@ sub _update_batch_record_counts { num_records = ( SELECT COUNT(*) FROM import_records - WHERE import_batch_id = import_batches.import_batch_id - AND record_type = 'biblio'), + WHERE import_batch_id = import_batches.import_batch_id), num_items = ( SELECT COUNT(*) FROM import_records @@ -1432,7 +1505,15 @@ sub _get_commit_action { } elsif ($overlay_action eq 'ignore') { $bib_result = 'ignore'; } - $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore'; + if($item_action eq 'always_add' or $item_action eq 'add_only_for_matches'){ + $item_result = 'create_new'; + } + elsif($item_action eq 'replace'){ + $item_result = 'replace'; + } + else { + $item_result = 'ignore'; + } } else { $bib_result = $nomatch_action; $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore';