9 use base qw/WebPAC::Common/;
14 WebPAC::Input - read different file formats into WebPAC
22 our $VERSION = '0.03';
26 This module implements input as database which have fixed and known
27 I<size> while indexing and single unique numeric identifier for database
28 position ranging from 1 to I<size>.
30 Simply, something that is indexed by unmber from 1 .. I<size>.
32 Examples of such databases are CDS/ISIS files, MARC files, lines in
35 Specific file formats are implemented using low-level interface modules,
36 located in C<WebPAC::Input::*> namespace which export C<open_db>,
37 C<fetch_rec> and optional C<init> functions.
39 Perhaps a little code snippet.
43 my $db = WebPAC::Input->new(
44 module => 'WebPAC::Input::ISIS',
46 lookup => $lookup_obj,
50 $db->open('/path/to/database');
51 print "database size: ",$db->size,"\n";
52 while (my $rec = $db->fetch) {
61 Create new input database object.
63 my $db = new WebPAC::Input(
64 module => 'WebPAC::Input::MARC',
65 code_page => 'ISO-8859-2',
69 C<module> is low-level file format module. See L<WebPAC::Input::Isis> and
70 L<WebPAC::Input::MARC>.
72 Optional parametar C<code_page> specify application code page (which will be
73 used internally). This should probably be your terminal encoding, and by
74 default, it C<ISO-8859-2>.
76 Default is not to use C<low_mem> options (see L<MEMORY USAGE> below).
78 This function will also call low-level C<init> if it exists with same
88 my $log = $self->_get_logger;
90 $log->logconfess("specify low-level file format module") unless ($self->{module});
91 my $module = $self->{module};
94 $log->debug("require low-level module $self->{module} from $module");
97 #eval $self->{module} .'->import';
99 # check if required subclasses are implemented
100 foreach my $subclass (qw/open_db fetch_rec init/) {
101 my $n = $self->{module} . '::' . $subclass;
102 if (! defined &{ $n }) {
103 my $missing = "missing $subclass in $self->{module}";
104 $self->{$subclass} = sub { $log->logwarn($missing) };
106 $self->{$subclass} = \&{ $n };
111 $log->debug("calling init");
112 $self->{init}->($self, @_);
115 $self->{'code_page'} ||= 'ISO-8859-2';
117 # running with low_mem flag? well, use DBM::Deep then.
118 if ($self->{'low_mem'}) {
119 $log->info("running with low_mem which impacts performance (<32 Mb memory usage)");
121 my $db_file = "data.db";
124 unlink $db_file or $log->logdie("can't remove '$db_file' from last run");
125 $log->debug("removed '$db_file' from last run");
130 my $db = new DBM::Deep $db_file;
132 $log->logdie("DBM::Deep error: $!") unless ($db);
135 $log->logdie("can't open '$db_file' under low_mem: ",$db->error());
137 $log->debug("using file '$db_file' for DBM::Deep");
143 $self ? return $self : return undef;
148 This function will read whole database in memory and produce lookups.
151 path => '/path/to/database/file',
155 lookup => $lookup_obj,
158 By default, C<code_page> is assumed to be C<852>.
160 C<offset> is optional parametar to position at some offset before reading from database.
162 C<limit> is optional parametar to read just C<limit> records from database
164 Returns size of database, regardless of C<offset> and C<limit>
165 parametars, see also C<size>.
173 my $log = $self->_get_logger();
175 $log->logcroak("need path") if (! $arg->{'path'});
176 my $code_page = $arg->{'code_page'} || '852';
178 # store data in object
179 $self->{'input_code_page'} = $code_page;
180 foreach my $v (qw/path offset limit/) {
181 $self->{$v} = $arg->{$v} if ($arg->{$v});
184 # create Text::Iconv object
185 $self->{iconv} = Text::Iconv->new($code_page,$self->{'code_page'});
187 my ($db, $size) = $self->{open_db}->( $self,
188 path => $arg->{path},
192 $log->logwarn("can't open database $arg->{path}, skipping...");
197 $log->logwarn("no records in database $arg->{path}, skipping...");
204 if (my $s = $self->{offset}) {
205 $log->info("skipping to MFN $s");
208 $self->{offset} = $offset;
211 if ($self->{limit}) {
212 $log->debug("limiting to ",$self->{limit}," records");
213 $limit = $offset + $self->{limit} - 1;
214 $limit = $size if ($limit > $size);
217 # store size for later
218 $self->{size} = ($limit - $offset) ? ($limit - $offset + 1) : 0;
220 $log->info("processing $self->{size} records in $code_page, convert to $self->{code_page}");
223 for (my $pos = $offset; $pos <= $limit; $pos++) {
225 $log->debug("position: $pos\n");
227 my $rec = $self->{fetch_rec}->($self, $db, $pos );
230 $log->warn("record $pos empty? skipping...");
235 if ($self->{low_mem}) {
236 $self->{db}->put($pos, $rec);
238 $self->{data}->{$pos} = $rec;
242 $self->{'lookup'}->add( $rec ) if ($rec && $self->{'lookup'});
244 $self->progress_bar($pos,$limit);
249 $self->{last_pcnt} = 0;
251 # store max mfn and return it.
252 $self->{max_pos} = $limit;
253 $log->debug("max_pos: $limit");
260 Fetch next record from database. It will also displays progress bar.
262 my $rec = $isis->fetch;
264 Record from this function should probably go to C<data_structure> for
272 my $log = $self->_get_logger();
274 $log->logconfess("it seems that you didn't load database!") unless ($self->{pos});
276 if ($self->{pos} == -1) {
277 $self->{pos} = $self->{offset};
282 my $mfn = $self->{pos};
284 if ($mfn > $self->{max_pos}) {
285 $self->{pos} = $self->{max_pos};
286 $log->debug("at EOF");
290 $self->progress_bar($mfn,$self->{max_pos});
294 if ($self->{low_mem}) {
295 $rec = $self->{db}->get($mfn);
297 $rec = $self->{data}->{$mfn};
305 Returns current record number (MFN).
309 First record in database has position 1.
321 Returns number of records in database
325 Result from this function can be used to loop through all records
327 foreach my $mfn ( 1 ... $isis->size ) { ... }
329 because it takes into account C<offset> and C<limit>.
335 return $self->{size};
340 Seek to specified MFN in file.
344 First record in database has position 1.
350 my $pos = shift || return;
352 my $log = $self->_get_logger();
355 $log->warn("seek before first record");
357 } elsif ($pos > $self->{max_pos}) {
358 $log->warn("seek beyond last record");
359 $pos = $self->{max_pos};
362 return $self->{pos} = (($pos - 1) || -1);
368 C<low_mem> options is double-edged sword. If enabled, WebPAC
369 will run on memory constraint machines (which doesn't have enough
370 physical RAM to create memory structure for whole source database).
372 If your machine has 512Mb or more of RAM and database is around 10000 records,
373 memory shouldn't be an issue. If you don't have enough physical RAM, you
374 might consider using virtual memory (if your operating system is handling it
375 well, like on FreeBSD or Linux) instead of dropping to L<DBM::Deep> to handle
376 parsed structure of ISIS database (this is what C<low_mem> option does).
378 Hitting swap at end of reading source database is probably o.k. However,
379 hitting swap before 90% will dramatically decrease performance and you will
380 be better off with C<low_mem> and using rest of availble memory for
381 operating system disk cache (Linux is particuallary good about this).
382 However, every access to database record will require disk access, so
383 generation phase will be slower 10-100 times.
385 Parsed structures are essential - you just have option to trade RAM memory
386 (which is fast) for disk space (which is slow). Be sure to have planty of
387 disk space if you are using C<low_mem> and thus L<DBM::Deep>.
389 However, when WebPAC is running on desktop machines (or laptops :-), it's
390 highly undesireable for system to start swapping. Using C<low_mem> option can
391 reduce WecPAC memory usage to around 64Mb for same database with lookup
392 fields and sorted indexes which stay in RAM. Performance will suffer, but
393 memory usage will really be minimal. It might be also more confortable to
394 run WebPAC reniced on those machines.
399 Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>
401 =head1 COPYRIGHT & LICENSE
403 Copyright 2005 Dobrica Pavlinusic, All Rights Reserved.
405 This program is free software; you can redistribute it and/or modify it
406 under the same terms as Perl itself.
410 1; # End of WebPAC::Input