ported r260 from hidra branch: moved eval to parse_format.pm where it

[webpac] / all2xml.pl

diff --git a/all2xml.pl b/all2xml.pl
index cfb5169..7f5ba01 100755 (executable)
--- a/all2xml.pl
+++ b/all2xml.pl
@@ -228,7 +228,7 @@ sub data2xml {
                                        if (($#data+1) == $nr) {
                                                return sprintf($fmt,@data);
                                        } else {
-                                               print STDERR "mkformat: [$data] can't be split on [$format_delimiter] to $nr fields!\n";
+                                               #print STDERR "mkformat: [$data] can't be split on [$format_delimiter] to $nr fields!\n";
                                                return $data;
                                        }
                                } else {
@@ -257,7 +257,7 @@ sub data2xml {
                                                                $display = $new_display;
                                                                $cache->{lhash}->{$display} = $new_display;
                                                        } else {
-                                                               print STDERR "WARNING: lookup for '$display' didn't find anything.\n";
+#                                                              print STDERR "WARNING: lookup for '$display' didn't find anything.\n";
                                                                $display = "";
                                                                $cache->{lhash}->{$display} = $null;
                                                        }
@@ -278,36 +278,35 @@ sub data2xml {
                                }
                                # type="swish" ; field for swish
                                if ($swish) {
+                                       my $tmp = $swish;
                                        if ($filter && ($s || $se)) {
                                                no strict 'refs';
-                                               my $tmp = join(" ",&$filter($swish)) if ($s || $se);
-                                               $swish_data .= $tmp if ($s);
-                                               $swish_exact_data .= "xxbxx $tmp xxexx " if ($se && $tmp ne "");
-
-                                       } else {
-                                               $swish_data .= $swish if ($s);
-                                               $swish_exact_data .= "xxbxx $swish xxexx " if ($se && $swish ne "");
+                                               $tmp = join(" ",&$filter($tmp)) if ($s || $se);
                                        }
+
+                                       $swish_data .= $tmp if ($s && $tmp);
+                                       $swish_exact_data .= "xxbxx $tmp xxexx " if ($tmp && $tmp ne "" && $se);
                                }
 
                                # type="display" ; field for display
                                if ($d && $display) {
+                                       my $ldel = $delimiter;
                                        if ($line_delimiter && $display_data) {
-                                               $display_data .= $line_delimiter;
+                                               $ldel = $line_delimiter;
                                        }
                                        if ($filter) {
                                                no strict 'refs';
-                                               if ($display_data) {
-                                                       $display_data .= $delimiter.join($delimiter,mkformat($x,&$filter($display)));
-                                               } else {
-                                                       $display_data = join($delimiter,mkformat($x,&$filter($display)));
+                                               my @arr;
+                                               foreach my $tmp (&$filter($display)) {
+                                                       my $tmp2 = mkformat($x,$tmp);
+                                                       push @arr,$tmp2 if ($tmp2);
                                                }
+                                               $display_data .= $ldel if ($display_data && @arr);
+                                               $display_data .= join($delimiter,@arr);
                                        } else {
-                                               if ($display_data) {
-                                                       $display_data .= $delimiter.mkformat($x,$display);
-                                               } else {
-                                                       $display_data = mkformat($x,$display);
-                                               }
+                                               $display_data .= $ldel if ($display_data);
+                                               my $tmp = mkformat($x,$display);
+                                               $display_data .= $tmp if ($tmp);
                                        }
                                }
                                                
@@ -319,7 +318,7 @@ sub data2xml {
                                                no strict 'refs';
                                                $idisplay = &$filter($idisplay);
                                        }
-                                       push @index_data, $idisplay if (! $iterate_by_page);
+                                       push @index_data, $idisplay if ($idisplay && !$iterate_by_page);
                                }
 
                                # store fields in lookup
@@ -328,11 +327,21 @@ sub data2xml {
                                                if ($lookup_key) {
                                                        print STDERR "WARNING: try to redefine lookup_key (keys shouldn't be repeatable fields!)";
                                                } else {
-                                                       $lookup_key = $display;
+                                                       if ($filter) {
+                                                               no strict 'refs';
+                                                               $lookup_key = &$filter($display);
+                                                       } else {
+                                                               $lookup_key = $display;
+                                                       }
                                                }
                                        } elsif (lc($x->{type}) eq "lookup_val") {
                                                if ($lookup_key) {
-                                                       $lhash{$lookup_key} = $display;
+                                                       if ($filter) {
+                                                               no strict 'refs';
+                                                               $lhash{$lookup_key} = &$filter($display);
+                                                       } else {
+                                                               $lhash{$lookup_key} = $display;
+                                                       }
                                                } else {
                                                        print STDERR "WARNING: no lookup_key defined for  '$display'?";
                                                }
@@ -451,7 +460,8 @@ sub data2xml {
                        for (my $page=0; $page <= $nr_pages; $page++) {
                                my $display_data;
                                if ($cache->{format}->{$field}) {
-                                       $display_data=mkformat($cache->{format}->{$field},$cache->{display_data}->{$field}->[$page]);
+                                       my $tmp = mkformat($cache->{format}->{$field},$cache->{display_data}->{$field}->[$page]);
+                                       $display_data=$tmp if ($tmp);
                                } else {
                                        $display_data = $cache->{display_data}->{$field}->[$page];
                                }
@@ -577,6 +587,8 @@ foreach my $database ($cfg->Sections) {
                #tie %lhash, 'GDBM_File', $lookup_file, &GDBM_NEWDB, 0644;
                tie %lhash, 'TDB_File', $lookup_file, TDB_CLEAR_IF_FIRST, O_RDWR, 0644;
                print STDERR "creating lookup file '$lookup_file'\n";
+               # delete memory cache for lookup file
+               delete $cache->{lhash};
        }
 
        # open existing lookup file
@@ -713,8 +725,9 @@ print STDERR "using: $type...\n";
                print STDERR "\n";
 
        } elsif ($type_base eq "excel") {
-               use Spreadsheet::ParseExcel;
-               use Spreadsheet::ParseExcel::Utility qw(int2col);
+               require Spreadsheet::ParseExcel;
+               require Spreadsheet::ParseExcel::Utility;
+               import Spreadsheet::ParseExcel::Utility qw(int2col);
                
                $import2cp = Text::Iconv->new($config->{excel_codepage},$codepage);
                my $excel_file = $cfg -> val($database, 'excel_file') || die "$database doesn't have 'excel_file' defined!";
@@ -768,29 +781,32 @@ print STDERR "using: $type...\n";
                }
        } elsif ($type_base eq "marc") {
 
-               use MARC;
+               require MARC;
                
                $import2cp = Text::Iconv->new($config->{marc_codepage},$codepage);
                my $marc_file = $cfg -> val($database, 'marc_file') || die "$database doesn't have 'marc_file' defined!";
 
                # optional argument is format
-               my $format = x($config->{format}) || 'usmarc';
+               my $format = x($config->{marc_format}) || 'usmarc';
 
                print STDERR "Reading MARC file '$marc_file'\n";
 
                my $marc = new MARC;
                my $nr = $marc->openmarc({
                                file=>$marc_file, format=>$format
-                       }) || die "Can't open MARC file '$marc_file'";
+                       }) || die "Can't open MARC file '$marc_file' with format '$format'";
+
+               # read MARC file in memory
+               $marc->nextmarc(-1);
 
-               my $i=0;        # record nr.
+               my $max_rec = $marc->marc_count();
 
-               my $rec;
+               for(my $i=1; $i<=$max_rec; $i++) {
 
-               while ($marc->nextmarc(1)) {
+                       progress($i,$max_rec);
 
-                       # XXX
-                       fakeprogress($i++);
+                       # store value for marc_sf.pm
+                       $main::cache->{marc_record} = $i;
 
                        my $swishpath = $database."#".$i;
 
@@ -802,6 +818,9 @@ print STDERR "using: $type...\n";
                                print "Document-Type: XML\n\n$xml\n";
                        }
                }
+
+               print STDERR "\n";
+
        } elsif ($type_base eq "feed") {
 
                $import2cp = Text::Iconv->new($config->{feed_codepage},$codepage);