projects
/
MARC-Fast
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
return correct count for marc files with additional junk bytes at end
[MARC-Fast]
/
Fast.pm
diff --git
a/Fast.pm
b/Fast.pm
index
7f2a618
..
dea7a91
100644
(file)
--- a/
Fast.pm
+++ b/
Fast.pm
@@
-1,13
+1,13
@@
-
package MARC::Fast;
package MARC::Fast;
+
use strict;
use Carp;
use strict;
use Carp;
-use Data::Dump
er
;
+use Data::Dump
qw/dump/
;
BEGIN {
use Exporter ();
use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
BEGIN {
use Exporter ();
use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
- $VERSION = 0.0
3
;
+ $VERSION = 0.0
9
;
@ISA = qw (Exporter);
#Give a hoot don't pollute, do not export more than needed by default
@EXPORT = qw ();
@ISA = qw (Exporter);
#Give a hoot don't pollute, do not export more than needed by default
@EXPORT = qw ();
@@
-23,12
+23,21
@@
MARC::Fast - Very fast implementation of MARC database reader
use MARC::Fast;
use MARC::Fast;
+ my $marc = new MARC::Fast(
+ marcdb => 'unimarc.iso',
+ );
+
+ foreach my $mfn ( 1 .. $marc->count ) {
+ print $marc->to_ascii( $mfn );
+ }
+
+For longer example with command line options look at L<scripts/dump_fastmarc.pl>
=head1 DESCRIPTION
This is very fast alternative to C<MARC> and C<MARC::Record> modules.
=head1 DESCRIPTION
This is very fast alternative to C<MARC> and C<MARC::Record> modules.
-It's is also very sutable for random access to MARC records (as opposed to
+It's is also very su
b
table for random access to MARC records (as opposed to
sequential one).
=head1 METHODS
sequential one).
=head1 METHODS
@@
-43,7
+52,7
@@
Read MARC database
debug => 0,
assert => 0,
hash_filter => sub {
debug => 0,
assert => 0,
hash_filter => sub {
- my
$t = shift
;
+ my
($t, $record_number) = @_
;
$t =~ s/foo/bar/;
return $t;
},
$t =~ s/foo/bar/;
return $t;
},
@@
-79,6
+88,7
@@
sub new {
if ($len < 24) {
carp "short read of leader, aborting\n";
if ($len < 24) {
carp "short read of leader, aborting\n";
+ $self->{count}--;
last;
}
last;
}
@@
-111,10
+121,11
@@
sub new {
print STDERR "REC ",$self->{count},": $leader\n" if ($self->{debug});
# store leader for later
print STDERR "REC ",$self->{count},": $leader\n" if ($self->{debug});
# store leader for later
- push @{$self->{leader
s
}}, $leader;
+ push @{$self->{leader}}, $leader;
# skip to next record
my $o = substr($leader,0,5);
# skip to next record
my $o = substr($leader,0,5);
+ warn "# in record ", $self->{count}," record length isn't number but: ",dump($o),"\n" unless $o =~ m/^\d+$/;
if ($o > 24) {
seek($self->{fh},$o-24,1) if ($o);
} else {
if ($o > 24) {
seek($self->{fh},$o-24,1) if ($o);
} else {
@@
-145,14
+156,22
@@
Fetch record from database
my $hash = $marc->fetch(42);
my $hash = $marc->fetch(42);
+First record number is C<1>
+
=cut
sub fetch {
my $self = shift;
=cut
sub fetch {
my $self = shift;
- my $rec_nr = shift || return;
+ my $rec_nr = shift;
+
+ if ( ! $rec_nr ) {
+ $self->{last_leader} = undef;
+ return;
+ }
- my $leader = $self->{leaders}->[$rec_nr - 1];
+ my $leader = $self->{leader}->[$rec_nr - 1];
+ $self->{last_leader} = $leader;
unless ($leader) {
carp "can't find record $rec_nr";
return;
unless ($leader) {
carp "can't find record $rec_nr";
return;
@@
-234,11
+253,31
@@
sub fetch {
}
}
+=head2 last_leader
+
+Returns leader of last record L<fetch>ed
+
+ print $marc->last_leader;
+
+Added in version 0.08 of this module, so if you need it use:
+
+ use MARC::Fast 0.08;
+
+to be sure that it's supported.
+
+=cut
+
+sub last_leader {
+ my $self = shift;
+ return $self->{last_leader};
+}
+
+
=head2 to_hash
Read record with specified MFN and convert it to hash
=head2 to_hash
Read record with specified MFN and convert it to hash
- my $hash = $marc->to_hash(
$mfn
);
+ my $hash = $marc->to_hash(
$mfn, include_subfields => 1,
);
It has ability to convert characters (using C<hash_filter>) from MARC
database before creating structures enabling character re-mapping or quick
It has ability to convert characters (using C<hash_filter>) from MARC
database before creating structures enabling character re-mapping or quick
@@
-265,65
+304,107
@@
sub to_hash {
my $mfn = shift || confess "need mfn!";
my $mfn = shift || confess "need mfn!";
+ my $args = {@_};
+
# init record to include MFN as field 000
my $rec = { '000' => [ $mfn ] };
my $row = $self->fetch($mfn) || return;
# init record to include MFN as field 000
my $rec = { '000' => [ $mfn ] };
my $row = $self->fetch($mfn) || return;
- foreach my $
k
(keys %{$row}) {
- foreach my $l (@{$row->{$
k
}}) {
+ foreach my $
rec_nr
(keys %{$row}) {
+ foreach my $l (@{$row->{$
rec_nr
}}) {
# remove end marker
$l =~ s/\x1E$//;
# filter output
# remove end marker
$l =~ s/\x1E$//;
# filter output
- $l = $self->{'hash_filter'}->($l) if ($self->{'hash_filter'});
+ $l = $self->{'hash_filter'}->($l
, $rec_nr
) if ($self->{'hash_filter'});
my $val;
# has identifiers?
($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])\x1F/\x1F/);
my $val;
# has identifiers?
($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])\x1F/\x1F/);
+ my $sf_usage;
+ my @subfields;
+
# has subfields?
if ($l =~ m/\x1F/) {
foreach my $t (split(/\x1F/,$l)) {
next if (! $t);
my $f = substr($t,0,1);
# has subfields?
if ($l =~ m/\x1F/) {
foreach my $t (split(/\x1F/,$l)) {
next if (! $t);
my $f = substr($t,0,1);
- # repeatable subfileds. When we hit first one,
- # store CURRENT (up to that) in first repetition
- # of this record. Then, new record with same
- #
identifiers will be created.
+
+ push @subfields, ( $f, $sf_usage->{$f}++ || 0 );
+
+ #
repeatable subfiled -- convert it to array
if ($val->{$f}) {
if ($val->{$f}) {
- push @{$rec->{$k}}, $val;
- $val = {
- i1 => $val->{i1},
- i2 => $val->{i2},
- }
;
+ if ( ref($val->{$f}) ne 'ARRAY' ) {
+ $val->{$f} = [ $val->{$f}, $val ];
+ } else {
+ push @{$val->{$f}}, $val;
+ }
}
$val->{substr($t,0,1)} = substr($t,1);
}
}
$val->{substr($t,0,1)} = substr($t,1);
}
+ $val->{subfields} = [ @subfields ] if $args->{include_subfields};
} else {
$val = $l;
}
} else {
$val = $l;
}
- push @{$rec->{$
k
}}, $val;
+ push @{$rec->{$
rec_nr
}}, $val;
}
}
return $rec;
}
}
}
return $rec;
}
+=head2 to_ascii
+
+ print $marc->to_ascii( 42 );
+
+=cut
+
+sub to_ascii {
+ my $self = shift;
+
+ my $mfn = shift || confess "need mfn";
+ my $row = $self->fetch($mfn) || return;
+
+ my $out;
+
+ foreach my $f (sort keys %{$row}) {
+ my $dump = join('', @{ $row->{$f} });
+ $dump =~ s/\x1e$//;
+ $dump =~ s/\x1f/\$/g;
+ $out .= "$f\t$dump\n";
+ }
+
+ return $out;
+}
1;
__END__
1;
__END__
-=head1
BUGS
+=head1
UTF-8 ENCODING
+This module does nothing with encoding. But, since MARC format is byte
+oriented even when using UTF-8 which has variable number of bytes for each
+character, file is opened in binary mode.
+As a result, all scalars recturned to perl don't have utf-8 flag. Solution is
+to use C<hash_filter> and L<Encode> to decode utf-8 encoding like this:
-=head1 SUPPORT
+ use Encode;
+ my $marc = new MARC::Fast(
+ marcdb => 'utf8.marc',
+ hash_filter => sub {
+ Encode::decode( 'utf-8', $_[0] );
+ },
+ );
+This will affect C<to_hash>, but C<fetch> will still return binary representation
+since it doesn't support C<hash_filter>.
=head1 AUTHOR
=head1 AUTHOR
@@
-343,6
+424,6
@@
LICENSE file included with this module.
=head1 SEE ALSO
=head1 SEE ALSO
-perl(1).
+
L<Biblio::Isis>,
perl(1).
=cut
=cut