#!/usr/bin/perl
# Import an iso2709 file into Koha 3
-use strict;
-use warnings;
+use Modern::Perl;
#use diagnostics;
BEGIN {
# find Koha's Perl modules
use C4::Debug;
use C4::Charset;
use C4::Items;
+use C4::MarcModificationTemplates;
+
use YAML;
use Unicode::Normalize;
use Time::HiRes qw(gettimeofday);
use IO::File;
use Pod::Usage;
+use Koha::Biblios;
+use Koha::SearchEngine;
+use Koha::SearchEngine::Search;
+
use open qw( :std :encoding(UTF-8) );
binmode( STDOUT, ":encoding(UTF-8)" );
my ( $input_marc_file, $number, $offset) = ('',0,0);
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
-my ( $insert, $filters, $update, $all, $yamlfile, $authtypes );
+my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );
my $cleanisbn = 1;
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
my $framework = '';
+my $localcust;
+my $marc_mod_template = '';
+my $marc_mod_template_id = -1;
$|=1;
't|test' => \$test_parameter,
's' => \$skip_marc8_conversion,
'c:s' => \$char_encoding,
- 'v:s' => \$verbose,
+ 'v:+' => \$verbose,
'fk' => \$fk_off,
'm:s' => \$format,
'l:s' => \$logfile,
+ 'append' => \$append,
'k|keepids:s' => \$keepids,
'b|biblios' => \$biblios,
'a|authorities' => \$authorities,
'yaml:s' => \$yamlfile,
'dedupbarcode' => \$dedup_barcode,
'framework=s' => \$framework,
+ 'custom:s' => \$localcust,
+ 'marcmodtemplate:s' => \$marc_mod_template,
);
$biblios ||= !$authorities;
$insert ||= !$update;
+my $writemode = ($append) ? "a" : "w";
+
+pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval ) if $biblios && $authorities;
if ($all) {
$insert = 1;
pod2usage( -verbose => 2 );
exit;
}
+if( $update && !( $match || $isbn_check ) ) {
+ warn "Using -update without -match or -isbn seems to be useless.\n";
+}
+
+if(defined $localcust) { #local customize module
+ if(!-e $localcust) {
+ $localcust= $localcust||'LocalChanges'; #default name
+ $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only
+ $localcust=~ s/\.pm$//; #remove extension
+ my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir
+ if(-e $fqcust) {
+ $localcust= $fqcust;
+ }
+ else {
+ print "WARNING: customize module $localcust.pm not found!\n";
+ exit 1;
+ }
+ }
+ require $localcust if $localcust;
+ $localcust=\&customize if $localcust;
+}
+
+if($marc_mod_template ne '') {
+ my @templates = GetModificationTemplates();
+ foreach my $this_template (@templates) {
+ if($this_template->{'name'} eq $marc_mod_template) {
+ if($marc_mod_template_id < 0) {
+ $marc_mod_template_id = $this_template->{'template_id'};
+ } else {
+ print "WARNING: MARC modification template name " .
+ "'$marc_mod_template' matches multiple templates. " .
+ "Please rename these templates\n";
+ exit 1;
+ }
+ }
+ }
+ if($marc_mod_template_id < 0) {
+ die "Can't located MARC modification template '$marc_mod_template'\n";
+ } else {
+ print "Records will be modified using MARC modofication template: $marc_mod_template\n" if $verbose;
+ }
+}
my $dbh = C4::Context->dbh;
my $heading_fields=get_heading_fields();
$sourcesubfield="a";
}
-# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import &
-# will create problems in the action_logs table, that can't handle more than 1 entry per second per user.
-my $CataloguingLog = C4::Context->preference('CataloguingLog');
-$dbh->do("UPDATE systempreferences SET value=0 WHERE variable='CataloguingLog'");
+
+# Disable logging for the biblios and authorities import operation. It would unnecessarily
+# slow the import
+
+# Disable the syspref cache so we can change logging settings
+C4::Context->disable_syspref_cache();
+# Save current CataloguingLog and AuthoritiesLog sysprefs values
+my $CataloguingLog = C4::Context->preference( 'CataloguingLog' );
+my $AuthoritiesLog = C4::Context->preference( 'AuthoritiesLog' );
+# Disable logging for both
+C4::Context->set_preference( 'CataloguingLog', 0 );
+C4::Context->set_preference( 'AuthoritiesLog', 0 );
if ($fk_off) {
$dbh->do("SET FOREIGN_KEY_CHECKS = 0");
if ($delete) {
if ($biblios){
print "deleting biblios\n";
- $dbh->do("truncate biblio");
- $dbh->do("truncate biblioitems");
- $dbh->do("truncate items");
+ $dbh->do("DELETE FROM biblio");
+ $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
+ $dbh->do("DELETE FROM biblioitems");
+ $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
+ $dbh->do("DELETE FROM items");
+ $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
}
else {
print "deleting authorities\n";
$dbh->{AutoCommit} = 0;
my $loghandle;
if ($logfile){
- $loghandle= IO::File->new($logfile,"w") ;
+ $loghandle= IO::File->new($logfile, $writemode) ;
print $loghandle "id;operation;status\n";
}
+
+my $searcher = Koha::SearchEngine::Search->new(
+ {
+ index => (
+ $authorities
+ ? $Koha::SearchEngine::AUTHORITIES_INDEX
+ : $Koha::SearchEngine::BIBLIOS_INDEX
+ )
+ }
+);
+
RECORD: while ( ) {
my $record;
# get records
# skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure
last unless ( $record );
$i++;
- print ".";
- print "\n$i" unless $i % 100;
-
+ if( ($verbose//1)==1 ) { #no dot for verbose==2
+ print "." . ( $i % 100==0 ? "\n$i" : '' );
+ }
+
# transcode the record to UTF8 if needed & applicable.
if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
# FIXME update condition
}
}
SetUTF8Flag($record);
+ if($marc_mod_template_id > 0) {
+ print "Modifying MARC\n" if $verbose;
+ ModifyRecordWithTemplate( $marc_mod_template_id, $record );
+ }
+ &$localcust($record) if $localcust;
my $isbn;
# remove trailing - in isbn (only for biblios, of course)
- if ($biblios && $cleanisbn) {
+ if( $biblios ) {
my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
my $field = $record->field($tag);
- my $isbn = $field && $field->subfield('a');
- if ( $isbn ) {
+ $isbn = $field && $field->subfield('a');
+ if ( $isbn && $cleanisbn ) {
$isbn =~ s/-//g;
$field->update('a' => $isbn);
}
my $query = build_query( $match, $record );
my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
$debug && warn $query;
- my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] );
- die "unable to search the database for duplicates : $error" if ( defined $error );
+ my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
+ # changed to warn so able to continue with one broken record
+ if ( defined $error ) {
+ warn "unable to search the database for duplicates : $error";
+ printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);
+ next RECORD;
+ }
$debug && warn "$query $server : $totalhits";
if ( $results && scalar(@$results) == 1 ) {
- my $marcrecord = MARC::File::USMARC::decode( $results->[0] );
+ my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );
SetUTF8Flag($marcrecord);
$id = GetRecordId( $marcrecord, $tagid, $subfieldid );
if ( $authorities && $marcFlavour ) {
#Skip if authority in database is the same as the on in database
- if ( $marcrecord->field('005')->data >= $record->field('005')->data ) {
+ if ( $marcrecord->field('005') && $record->field('005') &&
+ $marcrecord->field('005')->data && $record->field('005')->data &&
+ $marcrecord->field('005')->data >= $record->field('005')->data ) {
if ($yamlfile) {
$yamlhash->{$originalid}->{'authid'} = $id;
}
# create biblio, unless we already have it ( either match or isbn )
if ($biblionumber) {
- eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
+ eval{
+ $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;
+ };
if ($update) {
- eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) };
+ eval { ModBiblio( $record, $biblionumber, GetFrameworkCode($biblionumber) ) };
if ($@) {
warn "ERROR: Edit biblio $biblionumber failed: $@\n";
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
}
} else {
+ warn "WARNING: Updating record ".($id||$originalid)." failed";
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
+ next RECORD;
}
}
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
next RECORD;
}
else{
- printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
+ printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
}
if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
# Find the record called 'barcode'
ModBiblioMarc( $record, $biblionumber, $framework );
next RECORD;
} else {
- printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
+ printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
}
push @$errors_ref, @{ $more_errors };
}
}
$dbh->commit() if (0 == $i % $commitnum);
}
+ print $record->as_formatted()."\n" if ($verbose//0)==2;
last if $i == $number;
}
$dbh->commit();
-
+$dbh->{AutoCommit} = 1;
if ($fk_off) {
$dbh->do("SET FOREIGN_KEY_CHECKS = 1");
}
-# restore CataloguingLog
-$dbh->do("UPDATE systempreferences SET value=$CataloguingLog WHERE variable='CataloguingLog'");
+# Restore CataloguingLog
+C4::Context->set_preference( 'CataloguingLog', $CataloguingLog );
+# Restore AuthoritiesLog
+C4::Context->set_preference( 'AuthoritiesLog', $AuthoritiesLog );
my $timeneeded = gettimeofday - $starttime;
print "\n$i MARC records done in $timeneeded seconds\n";
File logs actions done for each record and their status into file
+=item B<-append>
+
+If specified, data will be appended to the logfile. If not, the logfile will be erased for each execution.
+
=item B<-t, -test>
-Test mode: parses the file, saying what he would do, but doing nothing.
+Test mode: parses the file, saying what it would do, but doing nothing.
=item B<-s>
=item B<-k, -keepids>=<FIELD>
-Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
+Field store ids in I<FIELD> (useful for authorities, where 001 contains the
authid for Koha, that can contain a very valuable info for authorities coming
from LOC or BNF. useless for biblios probably)
=item B<-keepids>
-Store ids in 009 (usefull for authorities, where 001 contains the authid for
+Store ids in 009 (useful for authorities, where 001 contains the authid for
Koha, that can contain a very valuable info for authorities coming from LOC or
BNF. useless for biblios probably)
to them when they are created. If not specified, then the default framework
will be used.
+=item B<-custom>=I<MODULE>
+
+This parameter allows you to use a local module with a customize subroutine
+that is called for each MARC record.
+If no filename is passed, LocalChanges.pm is assumed to be in the
+migration_tools subdirectory. You may pass an absolute file name or a file name
+from the migration_tools directory.
+
+=item B<-marcmodtemplate>=I<TEMPLATE>
+
+This parameter allows you to specify the name of an existing MARC
+modification template to apply as the MARC records are imported (these
+templates are created in the "MARC modification templates" tool in Koha).
+If not specified, no MARC modification templates are used (default).
+
=back
=cut
projects / koha.git / blobdiff