9 use base qw/WebPAC::Common/;
14 WebPAC::Input - read different file formats into WebPAC
22 our $VERSION = '0.03';
26 This module implements input as database which have fixed and known
27 I<size> while indexing and single unique numeric identifier for database
28 position ranging from 1 to I<size>.
30 Simply, something that is indexed by unmber from 1 .. I<size>.
32 Examples of such databases are CDS/ISIS files, MARC files, lines in
35 Specific file formats are implemented using low-level interface modules,
36 located in C<WebPAC::Input::*> namespace which export C<open_db>,
37 C<fetch_rec> and optional C<init> functions.
39 Perhaps a little code snippet.
43 my $db = WebPAC::Input->new(
44 module => 'WebPAC::Input::ISIS',
46 lookup => $lookup_obj,
50 $db->open('/path/to/database');
51 print "database size: ",$db->size,"\n";
52 while (my $rec = $db->fetch) {
61 Create new input database object.
63 my $db = new WebPAC::Input(
64 module => 'WebPAC::Input::MARC',
65 code_page => 'ISO-8859-2',
69 C<module> is low-level file format module. See L<WebPAC::Input::Isis> and
70 L<WebPAC::Input::MARC>.
72 Optional parametar C<code_page> specify application code page (which will be
73 used internally). This should probably be your terminal encoding, and by
74 default, it C<ISO-8859-2>.
76 Default is not to use C<low_mem> options (see L<MEMORY USAGE> below).
78 This function will also call low-level C<init> if it exists with same
88 my $log = $self->_get_logger;
90 $log->logconfess("specify low-level file format module") unless ($self->{module});
91 my $module = $self->{module};
94 $log->debug("require low-level module $self->{module} from $module");
97 #eval $self->{module} .'->import';
99 # check if required subclasses are implemented
100 foreach my $subclass (qw/open_db fetch_rec init/) {
101 my $n = $self->{module} . '::' . $subclass;
102 if (! defined &{ $n }) {
103 $log->logwarn("missing implementation of $subclass");
105 $self->{$subclass} = \&{ $n };
110 $log->debug("calling init");
111 $self->{init}->($self, @_);
114 $self->{'code_page'} ||= 'ISO-8859-2';
116 # running with low_mem flag? well, use DBM::Deep then.
117 if ($self->{'low_mem'}) {
118 $log->info("running with low_mem which impacts performance (<32 Mb memory usage)");
120 my $db_file = "data.db";
123 unlink $db_file or $log->logdie("can't remove '$db_file' from last run");
124 $log->debug("removed '$db_file' from last run");
129 my $db = new DBM::Deep $db_file;
131 $log->logdie("DBM::Deep error: $!") unless ($db);
134 $log->logdie("can't open '$db_file' under low_mem: ",$db->error());
136 $log->debug("using file '$db_file' for DBM::Deep");
142 $self ? return $self : return undef;
147 This function will read whole database in memory and produce lookups.
150 path => '/path/to/database/file',
154 lookup => $lookup_obj,
157 By default, C<code_page> is assumed to be C<852>.
159 C<offset> is optional parametar to position at some offset before reading from database.
161 C<limit> is optional parametar to read just C<limit> records from database
163 Returns size of database, regardless of C<offset> and C<limit>
164 parametars, see also C<size>.
172 my $log = $self->_get_logger();
174 $log->logcroak("need path") if (! $arg->{'path'});
175 my $code_page = $arg->{'code_page'} || '852';
177 # store data in object
178 $self->{'code_page'} = $code_page;
179 foreach my $v (qw/path offset limit/) {
180 $self->{$v} = $arg->{$v} if ($arg->{$v});
183 # create Text::Iconv object
184 $self->{iconv} = Text::Iconv->new($code_page,$self->{'code_page'});
186 my ($db, $size) = $self->{open_db}->( $self,
187 path => $arg->{path},
191 $log->logwarn("can't open database $arg->{path}, skipping...");
196 $log->logwarn("no records in database $arg->{path}, skipping...");
203 if (my $s = $self->{offset}) {
204 $log->info("skipping to MFN $s");
207 $self->{offset} = $offset;
210 if ($self->{limit}) {
211 $log->info("limiting to ",$self->{limit}," records");
212 $limit = $offset + $self->{limit} - 1;
213 $limit = $size if ($limit > $size);
216 # store size for later
217 $self->{size} = ($limit - $offset) ? ($limit - $offset + 1) : 0;
219 $log->info("processing $self->{size} records in $code_page, convert to $self->{code_page}");
222 for (my $pos = $offset; $pos <= $limit; $pos++) {
224 $log->debug("position: $pos\n");
226 my $rec = $self->{fetch_rec}->($self, $db, $pos );
229 $log->warn("record $pos empty? skipping...");
234 if ($self->{low_mem}) {
235 $self->{db}->put($pos, $rec);
237 $self->{data}->{$pos} = $rec;
241 $self->{'lookup'}->add( $rec ) if ($rec && $self->{'lookup'});
243 $self->progress_bar($pos,$limit);
248 $self->{last_pcnt} = 0;
250 # store max mfn and return it.
251 $self->{max_pos} = $limit;
252 $log->debug("max_pos: $limit");
259 Fetch next record from database. It will also displays progress bar.
261 my $rec = $isis->fetch;
263 Record from this function should probably go to C<data_structure> for
271 my $log = $self->_get_logger();
273 $log->logconfess("it seems that you didn't load database!") unless ($self->{pos});
275 if ($self->{pos} == -1) {
276 $self->{pos} = $self->{offset};
281 my $mfn = $self->{pos};
283 if ($mfn > $self->{max_pos}) {
284 $self->{pos} = $self->{max_pos};
285 $log->debug("at EOF");
289 $self->progress_bar($mfn,$self->{max_pos});
293 if ($self->{low_mem}) {
294 $rec = $self->{db}->get($mfn);
296 $rec = $self->{data}->{$mfn};
304 Returns current record number (MFN).
308 First record in database has position 1.
320 Returns number of records in database
324 Result from this function can be used to loop through all records
326 foreach my $mfn ( 1 ... $isis->size ) { ... }
328 because it takes into account C<offset> and C<limit>.
334 return $self->{size};
339 Seek to specified MFN in file.
343 First record in database has position 1.
349 my $pos = shift || return;
351 my $log = $self->_get_logger();
354 $log->warn("seek before first record");
356 } elsif ($pos > $self->{max_pos}) {
357 $log->warn("seek beyond last record");
358 $pos = $self->{max_pos};
361 return $self->{pos} = (($pos - 1) || -1);
367 C<low_mem> options is double-edged sword. If enabled, WebPAC
368 will run on memory constraint machines (which doesn't have enough
369 physical RAM to create memory structure for whole source database).
371 If your machine has 512Mb or more of RAM and database is around 10000 records,
372 memory shouldn't be an issue. If you don't have enough physical RAM, you
373 might consider using virtual memory (if your operating system is handling it
374 well, like on FreeBSD or Linux) instead of dropping to L<DBM::Deep> to handle
375 parsed structure of ISIS database (this is what C<low_mem> option does).
377 Hitting swap at end of reading source database is probably o.k. However,
378 hitting swap before 90% will dramatically decrease performance and you will
379 be better off with C<low_mem> and using rest of availble memory for
380 operating system disk cache (Linux is particuallary good about this).
381 However, every access to database record will require disk access, so
382 generation phase will be slower 10-100 times.
384 Parsed structures are essential - you just have option to trade RAM memory
385 (which is fast) for disk space (which is slow). Be sure to have planty of
386 disk space if you are using C<low_mem> and thus L<DBM::Deep>.
388 However, when WebPAC is running on desktop machines (or laptops :-), it's
389 highly undesireable for system to start swapping. Using C<low_mem> option can
390 reduce WecPAC memory usage to around 64Mb for same database with lookup
391 fields and sorted indexes which stay in RAM. Performance will suffer, but
392 memory usage will really be minimal. It might be also more confortable to
393 run WebPAC reniced on those machines.
398 Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>
400 =head1 COPYRIGHT & LICENSE
402 Copyright 2005 Dobrica Pavlinusic, All Rights Reserved.
404 This program is free software; you can redistribute it and/or modify it
405 under the same terms as Perl itself.
409 1; # End of WebPAC::Input