9 use base qw/WebPAC::Common/;
14 WebPAC::Input - read different file formats into WebPAC
22 our $VERSION = '0.03';
26 This module implements input as database which have fixed and known
27 I<size> while indexing and single unique numeric identifier for database
28 position ranging from 1 to I<size>.
30 Simply, something that is indexed by unmber from 1 .. I<size>.
32 Examples of such databases are CDS/ISIS files, MARC files, lines in
35 Specific file formats are implemented using low-level interface modules,
36 located in C<WebPAC::Input::*> namespace which export C<open_db>,
37 C<fetch_rec> and optional C<init> functions.
39 Perhaps a little code snippet.
43 my $db = WebPAC::Input->new(
44 module => 'WebPAC::Input::ISIS',
46 lookup => $lookup_obj,
50 $db->open('/path/to/database');
51 print "database size: ",$db->size,"\n";
52 while (my $rec = $db->fetch) {
61 Create new input database object.
63 my $db = new WebPAC::Input(
64 module => 'WebPAC::Input::MARC',
65 code_page => 'ISO-8859-2',
69 C<module> is low-level file format module. See L<WebPAC::Input::Isis> and
70 L<WebPAC::Input::MARC>.
72 Optional parametar C<code_page> specify application code page (which will be
73 used internally). This should probably be your terminal encoding, and by
74 default, it C<ISO-8859-2>.
76 Default is not to use C<low_mem> options (see L<MEMORY USAGE> below).
78 This function will also call low-level C<init> if it exists with same
88 my $log = $self->_get_logger;
90 $log->logconfess("specify low-level file format module") unless ($self->{module});
91 my $module = $self->{module};
94 $log->debug("require low-level module $self->{module} from $module");
96 eval $self->{module} .'->import';
98 # check if required subclasses are implemented
99 foreach my $subclass (qw/open_db fetch_rec/) {
100 if ( $self->can($subclass) ) {
101 $log->debug("imported $subclass");
103 $log->warn("missing $subclass in $self->{module}");
107 if ($self->can('init')) {
108 $log->debug("calling init");
112 $self->{'code_page'} ||= 'ISO-8859-2';
114 # running with low_mem flag? well, use DBM::Deep then.
115 if ($self->{'low_mem'}) {
116 $log->info("running with low_mem which impacts performance (<32 Mb memory usage)");
118 my $db_file = "data.db";
121 unlink $db_file or $log->logdie("can't remove '$db_file' from last run");
122 $log->debug("removed '$db_file' from last run");
127 my $db = new DBM::Deep $db_file;
129 $log->logdie("DBM::Deep error: $!") unless ($db);
132 $log->logdie("can't open '$db_file' under low_mem: ",$db->error());
134 $log->debug("using file '$db_file' for DBM::Deep");
140 $self ? return $self : return undef;
145 This function will read whole database in memory and produce lookups.
148 path => '/path/to/database/file',
152 lookup => $lookup_obj,
155 By default, C<code_page> is assumed to be C<852>.
157 C<offset> is optional parametar to position at some offset before reading from database.
159 C<limit> is optional parametar to read just C<limit> records from database
161 Returns size of database, regardless of C<offset> and C<limit>
162 parametars, see also C<size>.
170 my $log = $self->_get_logger();
172 $log->logcroak("need path") if (! $arg->{'path'});
173 my $code_page = $arg->{'code_page'} || '852';
175 # store data in object
176 $self->{'code_page'} = $code_page;
177 foreach my $v (qw/path offset limit/) {
178 $self->{$v} = $arg->{$v} if ($arg->{$v});
181 # create Text::Iconv object
182 $self->{iconv} = Text::Iconv->new($code_page,$self->{'code_page'});
184 my ($db, $size) = $self->open_db(
185 path => $arg->{path},
189 $log->logwarn("can't open database $arg->{path}, skipping...");
194 $log->logwarn("no records in database $arg->{path}, skipping...");
201 if (my $s = $self->{offset}) {
202 $log->info("skipping to MFN $s");
205 $self->{offset} = $offset;
208 if ($self->{limit}) {
209 $log->info("limiting to ",$self->{limit}," records");
210 $limit = $offset + $self->{limit} - 1;
211 $limit = $size if ($limit > $size);
214 # store size for later
215 $self->{size} = ($limit - $offset) ? ($limit - $offset + 1) : 0;
217 $log->info("processing $self->{size} records in $code_page, convert to $self->{code_page}");
220 for (my $pos = $offset; $pos <= $limit; $pos++) {
222 $log->debug("position: $pos\n");
224 my $rec = $self->fetch_rec( $db, $pos );
227 $log->warn("record $pos empty? skipping...");
232 if ($self->{low_mem}) {
233 $self->{db}->put($pos, $rec);
235 $self->{data}->{$pos} = $rec;
239 $self->{'lookup'}->add( $rec ) if ($rec && $self->{'lookup'});
241 $self->progress_bar($pos,$limit);
246 $self->{last_pcnt} = 0;
248 # store max mfn and return it.
249 $self->{max_pos} = $limit;
250 $log->debug("max_pos: $limit");
257 Fetch next record from database. It will also displays progress bar.
259 my $rec = $isis->fetch;
261 Record from this function should probably go to C<data_structure> for
269 my $log = $self->_get_logger();
271 $log->logconfess("it seems that you didn't load database!") unless ($self->{pos});
273 if ($self->{pos} == -1) {
274 $self->{pos} = $self->{offset};
279 my $mfn = $self->{pos};
281 if ($mfn > $self->{max_pos}) {
282 $self->{pos} = $self->{max_pos};
283 $log->debug("at EOF");
287 $self->progress_bar($mfn,$self->{max_pos});
291 if ($self->{low_mem}) {
292 $rec = $self->{db}->get($mfn);
294 $rec = $self->{data}->{$mfn};
302 Returns current record number (MFN).
306 First record in database has position 1.
318 Returns number of records in database
322 Result from this function can be used to loop through all records
324 foreach my $mfn ( 1 ... $isis->size ) { ... }
326 because it takes into account C<offset> and C<limit>.
332 return $self->{size};
337 Seek to specified MFN in file.
341 First record in database has position 1.
347 my $pos = shift || return;
349 my $log = $self->_get_logger();
352 $log->warn("seek before first record");
354 } elsif ($pos > $self->{max_pos}) {
355 $log->warn("seek beyond last record");
356 $pos = $self->{max_pos};
359 return $self->{pos} = (($pos - 1) || -1);
365 C<low_mem> options is double-edged sword. If enabled, WebPAC
366 will run on memory constraint machines (which doesn't have enough
367 physical RAM to create memory structure for whole source database).
369 If your machine has 512Mb or more of RAM and database is around 10000 records,
370 memory shouldn't be an issue. If you don't have enough physical RAM, you
371 might consider using virtual memory (if your operating system is handling it
372 well, like on FreeBSD or Linux) instead of dropping to L<DBM::Deep> to handle
373 parsed structure of ISIS database (this is what C<low_mem> option does).
375 Hitting swap at end of reading source database is probably o.k. However,
376 hitting swap before 90% will dramatically decrease performance and you will
377 be better off with C<low_mem> and using rest of availble memory for
378 operating system disk cache (Linux is particuallary good about this).
379 However, every access to database record will require disk access, so
380 generation phase will be slower 10-100 times.
382 Parsed structures are essential - you just have option to trade RAM memory
383 (which is fast) for disk space (which is slow). Be sure to have planty of
384 disk space if you are using C<low_mem> and thus L<DBM::Deep>.
386 However, when WebPAC is running on desktop machines (or laptops :-), it's
387 highly undesireable for system to start swapping. Using C<low_mem> option can
388 reduce WecPAC memory usage to around 64Mb for same database with lookup
389 fields and sorted indexes which stay in RAM. Performance will suffer, but
390 memory usage will really be minimal. It might be also more confortable to
391 run WebPAC reniced on those machines.
396 Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>
398 =head1 COPYRIGHT & LICENSE
400 Copyright 2005 Dobrica Pavlinusic, All Rights Reserved.
402 This program is free software; you can redistribute it and/or modify it
403 under the same terms as Perl itself.
407 1; # End of WebPAC::Input