use strict;
use Carp;
+use File::Glob qw(:globally :nocase);
+
use Data::Dumper;
BEGIN {
use Exporter ();
use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
- $VERSION = 0.06;
+ $VERSION = 0.09;
@ISA = qw (Exporter);
#Give a hoot don't pollute, do not export more than needed by default
@EXPORT = qw ();
isisdb => './cds/cds',
);
- for(my $mfn = 1; $mfn <= $isis->{'maxmfn'}; $mfn++) {
+ for(my $mfn = 1; $mfn <= $isis->count; $mfn++) {
print $isis->to_ascii($mfn),"\n";
}
=head1 DESCRIPTION
This module will read ISIS databases created by DOS CDS/ISIS, WinIsis or
-IsisMarc. It can be used as perl-only alternative to OpenIsis module.
+IsisMarc. It can be used as perl-only alternative to OpenIsis module which
+seems to depriciate it's old C<XS> bindings for perl.
It can create hash values from data in ISIS database (using C<to_hash>),
ASCII dump (using C<to_ascii>) or just hash with field names and packed
It also has support for identifiers (only if ISIS database is created by
IsisMarc), see C<to_hash>.
-This will module will always be slower than OpenIsis module which use C
+This module will always be slower than OpenIsis module which use C
library. However, since it's written in perl, it's platform independent (so
you don't need C compiler), and can be easily modified. I hope that it
creates data structures which are easier to use than ones created by
=item isisdb
This is full or relative path to ISIS database files which include
-common prefix of C<.FDT>, C<.MST>, C<.CNT>, C<.XRF> and C<.MST> files.
+common prefix of C<.MST>, and C<.XRF> and optionally C<.FDT> (if using
+C<read_fdt> option) files.
In this example it uses C<./cds/cds.MST> and related files.
=back
-It will also set C<$isis-E<gt>{'maxmfn'}> which is maximum MFN stored in database.
-
=cut
sub new {
$self->{$v} = {@_}->{$v};
}
+ my @isis_files = grep(/\.(FDT|MST|XRF|CNT)$/i,glob($self->{isisdb}."*"));
+
+ foreach my $f (@isis_files) {
+ my $ext = $1 if ($f =~ m/\.(\w\w\w)$/);
+ $self->{lc($ext)."_file"} = $f;
+ }
+
+ my @must_exist = qw(mst xrf);
+ push @must_exist, "fdt" if ($self->{read_fdt});
+
+ foreach my $ext (@must_exist) {
+ croak "missing ",uc($ext)," file in ",$self->{isisdb} unless ($self->{$ext."_file"});
+ }
+
+ print STDERR "## using files: ",join(" ",@isis_files),"\n" if ($self->{debug});
+
# if you want to read .FDT file use read_fdt argument when creating class!
- if ({@_}->{read_fdt} && -e $self->{isisdb}.".FDT") {
+ if ($self->{read_fdt} && -e $self->{fdt_file}) {
# read the $db.FDT file for tags
my $fieldzone=0;
- open(fileFDT, $self->{isisdb}.".FDT") || croak "can't read '$self->{isisdb}.FDT': $!";
+ open(fileFDT, $self->{fdt_file}) || croak "can't read '$self->{fdt_file}': $!";
while (<fileFDT>) {
chomp;
# Get the Maximum MFN from $db.MST
- open(fileMST,$self->{isisdb}.".MST") || croak "can't read '$self->{isisdb}.MST': $!";
+ open($self->{'fileMST'}, $self->{mst_file}) || croak "can't open '$self->{mst_file}': $!";
# MST format: (* = 32 bit signed)
# CTLMFN* always 0
# NXTMFB* last block allocated to master file
# NXTMFP offset to next available position in last block
# MFTYPE always 0 for user db file (1 for system)
- seek(fileMST,4,0);
+ seek($self->{'fileMST'},4,0);
my $buff;
- read(fileMST, $buff, 4);
+ read($self->{'fileMST'}, $buff, 4);
$self->{'NXTMFN'}=unpack("l",$buff) || carp "NXTNFN is zero";
- # save maximum MFN
- $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;
- close(fileMST);
- # Get the index information from $db.CNT
-
- open(fileCNT, $self->{isisdb}.".CNT") || croak "can't read '$self->{isisdb}.CNT': $!";
- # There is two 26 Bytes fixed lenght records
+ print STDERR Dumper($self),"\n" if ($self->{debug});
- # 0: IDTYPE BTree type 16
- # 2: ORDN Nodes Order 16
- # 4: ORDF Leafs Order 16
- # 6: N Number of Memory buffers for nodes 16
- # 8: K Number of buffers for first level index 16
- # 10: LIV Current number of Index Levels 16
- # 12: POSRX* Pointer to Root Record in N0x 32
- # 16: NMAXPOS* Next Available position in N0x 32
- # 20: FMAXPOS* Next available position in L0x 32
- # 24: ABNORMAL Formal BTree normality indicator 16
- # length: 26 bytes
+ # open files for later
+ open($self->{'fileXRF'}, $self->{xrf_file}) || croak "can't open '$self->{xrf_file}': $!";
- sub unpack_cnt {
- my $self = shift;
+ $self ? return $self : return undef;
+}
- my @flds = qw(ORDN ORDF N K LIV POSRX NMAXPOS FMAXPOS ABNORMAL);
+=head2 count
- my $buff = shift || return;
- my @arr = unpack("ssssssllls", $buff);
+Return number of records in database
- print STDERR "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});
+ print $isis->count;
- my $IDTYPE = shift @arr;
- foreach (@flds) {
- $self->{$IDTYPE}->{$_} = abs(shift @arr);
- }
- }
+=cut
+
+sub count {
+ my $self = shift;
+ return $self->{'NXTMFN'} - 1;
+}
+
+=head2 read_cnt
+
+Read content of C<.CNT> file and return hash containing it.
+
+ print Dumper($isis->read_cnt);
+
+This function is not used by module (C<.CNT> files are not required for this
+module to work), but it can be useful to examine your index (while debugging
+for example).
+
+=cut
+
+sub read_cnt {
+ my $self = shift;
+
+ croak "missing CNT file in ",$self->{isisdb} unless ($self->{cnt_file});
+
+ # Get the index information from $db.CNT
+
+ open(fileCNT, $self->{cnt_file}) || croak "can't read '$self->{cnt_file}': $!";
+
+ my $buff;
read(fileCNT, $buff, 26);
$self->unpack_cnt($buff);
read(fileCNT, $buff, 26);
$self->unpack_cnt($buff);
-
close(fileCNT);
- print STDERR Dumper($self),"\n" if ($self->{debug});
+ return $self->{cnt};
+}
- # open files for later
- open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";
+=head2 unpack_cnt
- open($self->{'fileMST'}, $self->{isisdb}.".MST") || croak "can't open '$self->{isisdb}.MST': $!";
+Unpack one of two 26 bytes fixed length record in C<.CNT> file.
- $self ? return $self : return undef;
+Here is definition of record:
+
+ off key description size
+ 0: IDTYPE BTree type s
+ 2: ORDN Nodes Order s
+ 4: ORDF Leafs Order s
+ 6: N Number of Memory buffers for nodes s
+ 8: K Number of buffers for first level index s
+ 10: LIV Current number of Index Levels s
+ 12: POSRX Pointer to Root Record in N0x l
+ 16: NMAXPOS Next Available position in N0x l
+ 20: FMAXPOS Next available position in L0x l
+ 24: ABNORMAL Formal BTree normality indicator s
+ length: 26 bytes
+
+This will fill C<$self> object under C<cnt> with hash. It's used by C<read_cnt>.
+
+=cut
+
+sub unpack_cnt {
+ my $self = shift;
+
+ my @flds = qw(ORDN ORDF N K LIV POSRX NMAXPOS FMAXPOS ABNORMAL);
+
+ my $buff = shift || return;
+ my @arr = unpack("ssssssllls", $buff);
+
+ print STDERR "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});
+
+ my $IDTYPE = shift @arr;
+ foreach (@flds) {
+ $self->{cnt}->{$IDTYPE}->{$_} = abs(shift @arr);
+ }
}
=head2 fetch
# is mfn allready in memory?
my $old_mfn = $self->{'current_mfn'} || -1;
- return if ($mfn == $old_mfn);
+ return $self->{record} if ($mfn == $old_mfn);
print STDERR "## fetch: $mfn\n" if ($self->{debug});
# XXX check this?
my $mfnpos=($mfn+int(($mfn-1)/127))*4;
- print STDERR "## seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});
+ print STDERR "## seeking to $mfnpos in file '$self->{xrf_file}'\n" if ($self->{debug});
seek($self->{'fileXRF'},$mfnpos,0);
my $buff;
+ # delete old record
+ delete $self->{record};
+
# read XRFMFB abd XRFMFP
read($self->{'fileXRF'}, $buff, 4);
my $pointer=unpack("l",$buff) || carp "pointer is null";
+ # check for logically deleted record
+ if ($pointer < 0) {
+ print STDERR "## record $mfn is logically deleted\n" if ($self->{debug});
+ $self->{deleted} = $mfn;
+
+ return unless $self->{include_deleted};
+
+ $pointer = abs($pointer);
+ }
+
my $XRFMFB = int($pointer/2048);
my $XRFMFP = $pointer - ($XRFMFB*2048);
-
# (XRFMFB - 1) * 512 + XRFMFP
# why do i have to do XRFMFP % 1024 ?
- my $blk_off = (($XRFMFB - 1) * 512) + ($XRFMFP % 1024);
+ my $blk_off = (($XRFMFB - 1) * 512) + ($XRFMFP % 512);
print STDERR "## pointer: $pointer XRFMFB: $XRFMFB XRFMFP: $XRFMFP offset: $blk_off\n" if ($self->{'debug'});
print STDERR "## offset for rowid $value is $blk_off (blk $XRFMFB off $XRFMFP)\n" if ($self->{debug});
if ($value!=$mfn) {
- carp "Error: MFN ".$mfn." not found in MST(".$value.")";
- #return; # XXX deleted record?
- }
+ if ($value == 0) {
+ print STDERR "## record $mfn is physically deleted\n" if ($self->{debug});
+ $self->{deleted} = $mfn;
+ return;
+ }
-# $MFRL=$self->Read16($fileMST);
-# $MFBWB=$self->Read32($fileMST);
-# $MFBWP=$self->Read16($fileMST);
-# $BASE=$self->Read16($fileMST);
-# $NVF=$self->Read16($fileMST);
-# $STATUS=$self->Read16($fileMST);
+ carp "Error: MFN ".$mfn." not found in MST file, found $value";
+ return;
+ }
read($self->{'fileMST'}, $buff, 14);
print STDERR "## MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});
- # delete old record
- delete $self->{record};
-
- ## FIXME this is a bug
- if (! $self->{'include_deleted'} && $MFRL < 0) {
- print "## logically deleted record $mfn, skipping...\n" if ($self->{debug});
- return;
- }
+ warn "MFRL $MFRL is not even number" unless ($MFRL % 2 == 0);
warn "BASE is not 18+6*NVF" unless ($BASE == 18 + 6 * $NVF);
for (my $i = 0 ; $i < $NVF ; $i++) {
-# $TAG=$self->Read16($fileMST);
-# $POS=$self->Read16($fileMST);
-# $LEN=$self->Read16($fileMST);
-
my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));
print STDERR "## TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});
push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);
}
- close(fileMST);
$self->{'current_mfn'} = $mfn;
- print Dumper($self),"\n" if ($self->{debug});
+ print STDERR Dumper($self),"\n" if ($self->{debug});
return $self->{'record'};
}
=head2 to_ascii
-Dump ASCII output of record with specified MFN
+Returns ASCII output of record with specified MFN
print $isis->to_ascii(42);
-It outputs something like this:
+This outputs something like this:
210 ^aNew York^cNew York University press^dcop. 1988
990 2140
my $hash = $isis->to_hash($mfn);
-It has ability to convert characters (using C<hash_filter> from ISIS
+It has ability to convert characters (using C<hash_filter>) from ISIS
database before creating structures enabling character re-mapping or quick
fix-up of data.
my $val;
# has identifiers?
- ($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])//);
+ ($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])\^/\^/);
# has subfields?
if ($l =~ m/\^/) {
=head1 BUGS
-This module has been very lightly tested. Use with caution and report bugs.
+Some parts of CDS/ISIS documentation are not detailed enough to exmplain
+some variations in input databases which has been tested with this module.
+When I was in doubt, I assumed that OpenIsis's implementation was right
+(except for obvious bugs).
+
+However, every effort has been made to test this module with as much
+databases (and programs that create them) as possible.
+
+I would be very greatful for success or failure reports about usage of this
+module with databases from programs other than WinIsis and IsisMarc. I had
+tested this against ouput of one C<isis.dll>-based application, but I don't
+know any details about it's version.
=head1 AUTHOR
projects / Biblio-Isis / blobdiff