BEGIN {
use Exporter ();
use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
- $VERSION = 0.01;
+ $VERSION = 0.04;
@ISA = qw (Exporter);
#Give a hoot don't pollute, do not export more than needed by default
@EXPORT = qw ();
=head1 SYNOPSIS
- use IsisDB
+ use IsisDB;
+
my $isis = new IsisDB(
isisdb => './cds/cds',
);
+ for(my $mfn = 1; $mfn <= $isis->{'maxmfn'}; $mfn++) {
+ print $isis->to_ascii($mfn),"\n";
+ }
+
=head1 DESCRIPTION
This module will read CDS/ISIS databases and create hash values out of it.
It can be used as perl-only alternative to OpenIsis module.
+This will module will always be slower that OpenIsis module which use C
+library. However, since it's written in perl, it's platform independent (so
+you don't need C compiler), and can be easily modified.
+
+Unique feature of this module is ability to C<include_deleted> records.
+It will also skip zero sized fields (OpenIsis has a bug in XS bindings, so
+fields which are zero sized will be filled with random junk from memory).
+
=head1 METHODS
=cut
# some binary reads
#
-sub Read32 {
- my $self = shift;
-
- my $f = shift || die "Read32 needs file handle";
- read($$f,$b,4) || die "can't read 4 bytes from $$f from position ".tell($f);
- return unpack("l",$b);
-}
-
=head2 new
Open CDS/ISIS database
my $isis = new IsisDB(
isisdb => './cds/cds',
read_fdt => 1,
+ include_deleted => 1,
+ hash_filter => sub {
+ my $v = shift;
+ $v =~ s#foo#bar#g;
+ },
debug => 1,
);
+Options are described below:
+
+=over 5
+
=item isisdb
Prefix path to CDS/ISIS. It should contain full or relative path to database
Boolean flag to specify if field definition table should be read. It's off
by default.
+=item include_deleted
+
+Don't skip logically deleted records in ISIS.
+
+=item hash_filter
+
+Filter code ref which will be used before data is converted to hash.
+
=item debug
-Dump a C<lot> of debugging output.
+Dump a B<lot> of debugging output.
+
+=back
+
+It will also set C<$isis-E<gt>{'maxmfn'}> which is maximum MFN stored in database.
=cut
my $self = {};
bless($self, $class);
- $self->{isisdb} = {@_}->{isisdb} || croak "new needs database name as argument!";
+ croak "new needs database name (isisdb) as argument!" unless ({@_}->{isisdb});
- $self->{debug} = {@_}->{debug} || 1; # XXX remove debug always!
+ foreach my $v (qw{isisdb debug include_deleted hash_filter}) {
+ $self->{$v} = {@_}->{$v};
+ }
# if you want to read .FDT file use read_fdt argument when creating class!
if ({@_}->{read_fdt} && -e $self->{isisdb}.".FDT") {
# NXTMFP offset to next available position in last block
# MFTYPE always 0 for user db file (1 for system)
seek(fileMST,4,0);
- $self->{'NXTMFN'}=$self->Read32(\*fileMST) || carp "NXTNFN is zero";
+
+ my $buff;
+
+ read(fileMST, $buff, 4);
+ $self->{'NXTMFN'}=unpack("l",$buff) || carp "NXTNFN is zero";
+
+ # save maximum MFN
+ $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;
close(fileMST);
my $buff = shift || return;
my @arr = unpack("ssssssllls", $buff);
+ print "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});
+
my $IDTYPE = shift @arr;
foreach (@flds) {
$self->{$IDTYPE}->{$_} = abs(shift @arr);
}
}
- my $buff;
read(fileCNT, $buff, 26);
$self->unpack_cnt($buff);
print Dumper($self) if ($self->{debug});
+ # open files for later
+ open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";
+
+ open($self->{'fileMST'}, $self->{isisdb}.".MST") || croak "can't open '$self->{isisdb}.MST': $!";
+
$self ? return $self : return undef;
}
+=head2 fetch
-# Get a record from the MFN
-# Return the number of fields in the record.
-# Return -1 if the record is marked for deletion
-# The record is then extracted with call to GETs
+Read record with selected MFN
-sub GetMFN {
- my $self = shift;
+ my $rec = $isis->fetch(55);
+
+Returns hash with keys which are field names and values are unpacked values
+for that field (like C<^asometing^bsomething else>)
- my $mfn = shift || croak "GetMFN needs MFN as argument!";
+=cut
- print "GetMFN: $mfn\n" if ($self->{debug});
+sub fetch {
+ my $self = shift;
- open(fileXRF, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";
+ my $mfn = shift || croak "fetch needs MFN as argument!";
+
+ print "fetch: $mfn\n" if ($self->{debug});
# XXX check this?
my $mfnpos=($mfn+int(($mfn-1)/127))*4;
print "seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});
- seek(fileXRF,$mfnpos,0);
+ seek($self->{'fileXRF'},$mfnpos,0);
+
+ my $buff;
# read XRFMFB abd XRFMFP
- my $pointer=$self->Read32(\*fileXRF);
+ read($self->{'fileXRF'}, $buff, 4);
+ my $pointer=unpack("l",$buff) || carp "pointer is null";
my $XRFMFB = int($pointer/2048);
my $XRFMFP = $pointer - ($XRFMFB*2048);
print "$offset - $offset2 - $offset3 - $offset4\n" if ($self->{debug});
- close(fileXRF);
-
# Get Record Information
- open(fileMST, $self->{isisdb}.".MST") || croak "can't open '$self->{isisdb}.MST': $!";
-
- seek(fileMST,$offset4,0);
+ seek($self->{'fileMST'},$offset4,0);
- my $value=$self->Read32(\*fileMST);
+ read($self->{'fileMST'}, $buff, 4);
+ my $value=unpack("l",$buff);
if ($value!=$mfn) {
print ("Error: The MFN:".$mfn." is not found in MST(".$value.")");
# $NVF=$self->Read16($fileMST);
# $STATUS=$self->Read16($fileMST);
- my $buff;
- read(fileMST, $buff, 14);
+ read($self->{'fileMST'}, $buff, 14);
my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);
print "MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});
+ # delete old record
+ delete $self->{record};
+
+ if (! $self->{'include_deleted'} && $MFRL < 0) {
+ print "## logically deleted record $mfn, skipping...\n" if ($self->{debug});
+ return;
+ }
+
# Get Directory Format
my @FieldPOS;
my @FieldLEN;
my @FieldTAG;
+ read($self->{'fileMST'}, $buff, 6 * $NVF);
+
+ my $fld_len = 0;
+
for (my $i = 0 ; $i < $NVF ; $i++) {
# $TAG=$self->Read16($fileMST);
# $POS=$self->Read16($fileMST);
# $LEN=$self->Read16($fileMST);
- read(fileMST, $buff, 6);
- my ($TAG,$POS,$LEN) = unpack("sss", $buff);
+ my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));
print "TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});
push @FieldTAG,$TAG;
push @FieldPOS,$POS;
push @FieldLEN,$LEN;
+
+ $fld_len += $LEN;
}
# Get Variable Fields
+ read($self->{'fileMST'},$buff,$fld_len);
+
for (my $i = 0 ; $i < $NVF ; $i++) {
- my $rec;
- read(fileMST,$rec,$FieldLEN[$i]);
- $self->{record}->{$FieldTAG[$i]} = $rec;
- }
- close(fileMST);
+ # skip zero-sized fields
+ next if ($FieldLEN[$i] == 0);
- # The record is marked for deletion
- if ($STATUS==1) {
- return -1;
+ push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);
}
+ close(fileMST);
print Dumper($self) if ($self->{debug});
- return $NVF;
+ return $self->{'record'};
}
-=begin php
-
- # Load the dictionary from the $db.L0x files.
- # Not usefull Yet
-
- sub LoadDictionary()
- {
- $fileL01=fopen($self->{isisdb}.".L01","r");
- rewind($fileL01);
-
- do
- {
-
- $POS=$self->Read32($fileL01);
- $OCK=$self->Read16($fileL01);
- $IT=$self->Read16($fileL01);
- $PS=$self->Read32($fileL01);
-print "<br>PS:".$PS." ".$self->{ORDF}->{1}." ";
- for ($i=0;$i<$OCK;$i++)
- {
- $KEY=fread($fileL01,10);
-
- print $KEY." ### ";
-
- $INFO1=$self->Read32($fileL01);
- $INFO2=$self->Read32($fileL01);
-
- #L01Key->{$key}=array($INFO1,$INFO2);
- }
-
- rewind($fileL01);
- $offset=($PS-1)*(12+$self->{ORDF}->{1}*18*2);
- fseek($fileL01,$offset);
-
- } While (!feof($fileL01));
-
- fclose($fileL01);
- }
+=head2 to_ascii
- # self function search through the tree and returns an array of pointers to IFP
- # The function must be recursive
-
- sub SearchTree($search,$fileNB,$PUNT)
- {
- $offset=(($PUNT-1)*(8+2*$self->{ORDN}->{1}*14));
-
- rewind($fileNB1);
-
- fseek($fileNB,$offset);
-
- $POS=$self->Read32($fileNB);
- $OCK=$self->Read16($fileNB);
- $IT=$self->Read16($fileNB);
-
-#print "<br>".$POS." - ".$OCK." - ".$IT;
-
- $OLDPUNT=$POS;
- $j=0;
- for ($i=0;$i<$OCK;$i++)
- {
- $KEY=fread($fileNB,10);
-
- $PUNT=$self->Read32($fileNB);
-
-#print " ## ".chop($KEY)."(".$PUNT."-".$OLDPUNT.") ## ";
-
- If (strcmp($search,chop($KEY))<0)
- {
- break;
- }
- $OLDPUNT=$PUNT;
- }
-#print $OLDPUNT;
- Return $OLDPUNT;
- }
+Dump ascii output of selected MFN
- # Search ISIS for record containing search
- # Return a sorted array of MFN
+ print $isis->to_ascii(55);
- sub Search($search)
- {
+=cut
- $search=strtoupper($search);
-#print "Searching....".$search." - ".$self->{POSRX}->{1}."<br>";
- # first search .x01
-
-
- # Search in .N01
-
-
- $fileN01=fopen($self->{isisdb}.".N01","r");
- $offset=(($self->{POSRX}->{1}-1)*(8+2*$self->{ORDN}->{1}*14));
-
- do
- {
- rewind($fileN01);
-
- fseek($fileN01,$offset);
-
- $POS=$self->Read32($fileN01);
- $OCK=$self->Read16($fileN01);
- $IT=$self->Read16($fileN01);
-
-#print "<br>".$POS." - ".$OCK." - ".$IT;
-
- $OLDPUNT=$POS;
- for ($i=0;$i<$OCK;$i++)
- {
- $KEY=fread($fileN01,10);
-
- $PUNT=$self->Read32($fileN01);
-
-#print " ## ".chop($KEY)."(".$PUNT."-".$OLDPUNT.") ## ";
-
- If (strcmp($search,chop($KEY))<0)
- {
- break;
- }
- $OLDPUNT=$PUNT;
- }
- $offset=(($OLDPUNT-1)*(8+2*$self->{ORDN}->{1}*14));
- } while ($OLDPUNT>0);
-#print $OLDPUNT;
-
-
- fclose($fileN01);
-
- # Now look for records in .L01 file
- $fileL01=fopen($self->{isisdb}.".L01","r");
- rewind($fileL01);
-
- $offset=(-$OLDPUNT-1)*(12+$self->{ORDF}->{1}*18*2);
- fseek($fileL01,$offset);
-
- $POS=$self->Read32($fileL01);
- $OCK=$self->Read16($fileL01);
- $IT=$self->Read16($fileL01);
- $PS=$self->Read32($fileL01);
-#print "<br>POS:".$POS." ".$self->{ORDF}->{1}." ";
- for ($i=0;$i<$OCK;$i++)
- {
- $KEY=fread($fileL01,10);
-
-#print $KEY." ### ";
-
- $INFO1=$self->Read32($fileL01);
- $INFO2=$self->Read32($fileL01);
-
- If (strcmp($search,chop($KEY))==0)
- {
- break;
- }
- }
-
- fclose($fileL01);
-
-#print $INFO1."--".$INFO2;
-
- # Now look in .IFP for the MFN
- $fileIFP=fopen($self->{isisdb}.".IFP","r");
- rewind($fileIFP);
- $offset=($INFO1-1)*512+($INFO2*4);
- fseek($fileIFP,$offset);
-
- $IFPBLK=$self->Read32($fileIFP);
-
- $IFPNXTB=$self->Read32($fileIFP);
- $IFPNXTP=$self->Read32($fileIFP);
- $IFPTOTP=$self->Read32($fileIFP);
- $IFPSEGP=$self->Read32($fileIFP);
- $IFPSEGC=$self->Read32($fileIFP);
-
-
-#print "<br>IFP:".$IFPBLK." # ".$IFPNXTB." - ".$IFPNXTP." - ".$IFPTOTP." - ".$IFPSEGP." - ".$IFPSEGC;
-
- rewind($fileIFP);
- $offset=($INFO1-1)*512+24+($INFO2*4);
- fseek($fileIFP,$offset);
-
- $j=24+($INFO2*4);
- $k=0;
- $l=1;
- $OLDPMFN="";
- for ($i=0;$i<$IFPSEGP;$i++)
- {
- $B1=$self->Read8($fileIFP);
- $B2=$self->Read8($fileIFP);
- $B3=$self->Read8($fileIFP);
- $B4=$self->Read8($fileIFP);
- $B5=$self->Read8($fileIFP);
- $B6=$self->Read8($fileIFP);
- $B7=$self->Read8($fileIFP);
- $B8=$self->Read8($fileIFP);
-
- $PMFN=$B1*65536+$B2*256+$B3;
- $PTAG=$B4*256+$B5;
- $POCC=$B6;
- $PCNT=$B7*256+$B8;
-
- if ($OLDPMFN!=$PMFN)
- {
- if ($PMFN!=0)
- {
- $self->{MFNArray}->{$l}=$PMFN;
- $OLDPMFN=$PMFN;
- $l+=1;
- }
- }
-
- $j=$j+8;
-#print "<br>".$PMFN."-".$PTAG." - ".$POCC." - ".$PCNT;
-#print "@@".$j."@@@@";
- if ($j>=504)
- {
- if ($IFPNXTB==0 && $IFPNXTP==0)
- {
- $k=$k+1;
- rewind($fileIFP);
- $offset=($INFO1-1+$k)*512;
- fseek($fileIFP,$offset);
- $B=$self->Read32($fileIFP);
-#print "<br>-".$B."-<br>";
- $j=0;
- } else
- {
- rewind($fileIFP);
- $offset=($IFPNXTB-1)*512;
- fseek($fileIFP,$offset);
-
- $OLDIFPNXTB=$IFPNXTB;
- $OLDIFPNXTP=$IFPNXTP;
-
- $IFPBLK=$self->Read32($fileIFP);
-
- $IFPNXTB=$self->Read32($fileIFP);
- $IFPNXTP=$self->Read32($fileIFP);
- $IFPTOTP=$self->Read32($fileIFP);
- $IFPSEGP=$self->Read32($fileIFP);
- $IFPSEGC=$self->Read32($fileIFP);
-
- rewind($fileIFP);
- $offset=($OLDIFPNXTB-1)*512+24+($OLDIFPNXTP*4);
- fseek($fileIFP,$offset);
-
- $j=24+($OLDIFPNXTP*4);
- $k=0;
- $j=0;
- }
- }
-
- }
- fclose($fileIFP);
- return $l-1;
- }
+sub to_ascii {
+ my $self = shift;
-=cut
+ my $mfn = shift || croak "need MFN";
-#
-# XXX porting from php left-over:
-#
-# do I *REALLY* need those methods, or should I use
-# $self->{something} directly?
-#
-# Probably direct usage is better!
-#
+ my $rec = $self->fetch($mfn);
-sub GetFieldName {
- my $self = shift;
- return $self->{FieldName};
-}
+ my $out = "0\t$mfn";
-sub GetTagName {
- my $self = shift;
- return $self->{TagName};
-}
+ foreach my $f (sort keys %{$rec}) {
+ $out .= "\n$f\t".join("\n$f\t",@{$self->{record}->{$f}});
+ }
-sub GetFieldTag {
- my $self = shift;
- return $self->{FieldTAG};
-}
+ $out .= "\n";
-sub GetNextMFN {
- my $self = shift;
- return $self->{NXTMFN};
+ return $out;
}
-sub GetMFNArray {
- my $self = shift;
- return $self->{MFNArray};
-}
-=begin php
-
- sub Read32($fileNB)
- {
- $B1=ord(fread($fileNB,1));
- $B2=ord(fread($fileNB,1));
- $B3=ord(fread($fileNB,1));
- $B4=ord(fread($fileNB,1));
-
- if ($B4<=128)
- {
- $value=$B1+$B2*256+$B3*65536+$B4*16777216;
- } else
- {
- $value=$self->Not8($B1)+$self->Not8($B2)*256+$self->Not8($B3)*65536+$self->Not8($B4)*16777216;
- $value=-($value+1);
- }
-# print "(".$B1.",".$B2.",".$B3.",".$B4.":".$value.")";
-
- return $value;
- }
+=head2 to_hash
- sub Read24($fileNB)
- {
- $B1=ord(fread($fileNB,1));
- $B2=ord(fread($fileNB,1));
- $B3=ord(fread($fileNB,1));
+Read mfn and convert it to hash
- $value=$B1+$B2*256+$B3*65536;
+ my $hash = $isis->to_hash($mfn);
-# print "(".$B1.",".$B2.",".$B3.":".$value.")";
+It has ability to convert characters (using C<hash_filter> from ISIS
+database before creating structures enabling character remapping or quick
+fixup of data.
- return $value;
- }
+This function returns hash which is like this:
- sub Read16($fileNB)
- {
- $B1=ord(fread($fileNB,1));
- $B2=ord(fread($fileNB,1));
+ $hash = {
+ '210' => [
+ {
+ 'c' => 'New York University press',
+ 'a' => 'New York',
+ 'd' => 'cop. 1988'
+ }
+ ],
+ '990' => [
+ '2140',
+ '88',
+ 'HAY'
+ ],
+ };
- $value=$B1+$B2*256;
-# print "(".$B1.",".$B2.":".$value.")";
+You can later use that has to produce any output from ISIS data.
- return $value;
- }
+=cut
- sub Read8($fileNB)
- {
- $B1=ord(fread($fileNB,1));
+sub to_hash {
+ my $self = shift;
- $value=$B1;
-# print "(".$value.")";
+ my $mfn = shift || confess "need mfn!";
- return $value;
- }
+ my $rec;
+ my $row = $self->fetch($mfn);
- sub Not8($value)
- {
- $value=decbin($value);
- if (strlen($value)<8)
- {
- $buffer="";
- for($i=0;$i<(8-strlen($value));$i++)
- {
- $buffer.="0";
- }
- $value=$buffer.$value;
- }
- $value=ereg_replace("0","3",$value);
- $value=ereg_replace("1","0",$value);
- $value=ereg_replace("3","1",$value);
- $value=bindec($value);
- return $value;
- }
+ foreach my $k (keys %{$row}) {
+ foreach my $l (@{$row->{$k}}) {
+
+ # filter output
+ $l = $self->{'hash_filter'}->($l) if ($self->{'hash_filter'});
+
+ # has subfields?
+ my $val;
+ if ($l =~ m/\^/) {
+ foreach my $t (split(/\^/,$l)) {
+ next if (! $t);
+ $val->{substr($t,0,1)} = substr($t,1);
+ }
+ } else {
+ $val = $l;
+ }
+
+ push @{$rec->{$k}}, $val;
+ }
+ }
+
+ return $rec;
}
-=cut
+#
+# XXX porting from php left-over:
+#
+# do I *REALLY* need those methods, or should I use
+# $self->{something} directly?
+#
+# Probably direct usage is better!
+#
+
+sub TagName {
+ my $self = shift;
+ return $self->{TagName};
+}
+
+sub NextMFN {
+ my $self = shift;
+ return $self->{NXTMFN};
+}
1;
-__END__
=head1 BUGS