major de-mungling of different codepages: use same codepage inside perl

(as opposed to UTF-8) and in files on disk

git-svn-id: file:///home/dpavlin/private/svn/webpac/trunk@40 13eb9ef6-21d5-0310-b721-a9d68796d827

diff --git a/all2xml.conf b/all2xml.conf
index 20f7898..ae221d8 100644 (file)
--- a/all2xml.conf
+++ b/all2xml.conf
@@ -7,8 +7,6 @@
        isis_db=/data/isis_data/small/ps/LIBRI/LIBRI
        # this will use xml file in import_xml/ dir
        type=isis
-       # this is codepage of import_xml/{type}.xml
-       xml_codepage = ISO8859-2
        # this is arbirary xml added to each record (e.g. Library identifier
        # or something like that
        xml=<type>Knjiga</type>
@@ -16,20 +14,17 @@
 [ge-libri]
        isis_db=/data/isis_data/small/ge/LIBRI/LIBRI
        type=isis
-       xml_codepage = ISO8859-2
        xml=<type>Knjiga</type>
 
 
 [ps-peri]
        isis_db=/data/isis_data/small/ps/PERI/PERI
        type=isis
-       xml_codepage = ISO8859-2
        xml=<type>Èasopis</type>
 
 [kk-arti]
        isis_db=/data/isis_data/small/kk/ARTI/ARTI
        type=isis
-       xml_codepage = ISO8859-2
        xml=<type>Èlanak</type>
 
 

diff --git a/all2xml.pl b/all2xml.pl
index 66fb8aa..d3b88f6 100755 (executable)
--- a/all2xml.pl
+++ b/all2xml.pl
@@ -8,6 +8,7 @@ use XML::Simple;
 use Text::Unaccent 1.02;       # 1.01 won't compile on my platform,
 use Text::Iconv;
 use Config::IniFiles;
+use Encode;
 
 $|=1;
 
@@ -33,20 +34,23 @@ getopts('d:m:qs', \%opts);
 
 my $db_dir;
 
-#die "usage: $0 -d [database_dir] -m [database1,database2] " if (! %opts);
-
-#print Dumper($config->{indexer});
-#print "-" x 70,"\n";
-
 Text::Iconv->raise_error(1);     # Conversion errors raise exceptions
 
-my $isis_codepage;
-my $index_codepage;
-my $cludge_codepage = Text::Iconv->new('UTF8','ISO8859-1');
-my $xml_codepage;
+# this is encoding of all files on disk, including import_xml/*.xml file and
+# filter/*.pm files! It will be used to store strings in perl internally!
+my $codepage = 'ISO-8859-2';
 
-my $XML_CHARSET = 'UTF8';
+my $utf2cp = Text::Iconv->new('UTF-8',$codepage);
+# this function will convert data from XML files to local encoding
+sub x {
+       return $utf2cp->convert($_[0]);
+}
 
+# decode isis import codepage
+my $isis2cp;
+
+# outgoing xml must be in UTF-8
+my $cp2utf = Text::Iconv->new($codepage,'UTF-8');
 
 sub isis2xml {
 
@@ -74,6 +78,8 @@ sub isis2xml {
 
        foreach my $field (sort by_order keys %{$config->{indexer}}) {
 
+               $field=x($field);
+
                $field_usage{$field}++;
 
                my $swish_data = "";
@@ -84,16 +90,8 @@ sub isis2xml {
 
                foreach my $x (@{$config->{indexer}->{$field}->{isis}}) {
 
-                       my $format = $x->{content};
-                       my $delimiter = $x->{delimiter} || ' ';
-
-                       # FIX: this is ugly, UGLY, cludge string is returned
-                       # in UTF8 encoding , but as if source charset
-                       # is ISO8859-1 and not some other. This break other
-                       # character encodings, so we convert it first
-                       # back to ISO8859-1
-                       $format = $xml_codepage->convert($format);
-                       $delimiter = $xml_codepage->convert($delimiter) if ($delimiter);
+                       my $format = x($x->{content});
+                       my $delimiter = x($x->{delimiter}) || ' ';
 
                        my $isis_i = 0;         # isis repeatable offset
 
@@ -101,7 +99,6 @@ sub isis2xml {
                        $s = 0 if (lc($x->{type}) eq "display");
                        $d = 0 if (lc($x->{type}) eq "swish");
                        ($s,$d,$i) = (0,0,1) if (lc($x->{type}) eq "index");
-#print STDERR "## s: $s d: $d i: $i ## $format ##\n";  
 
                        # what will separate last line from this one?
                        if ($display_data && $x->{append} && $x->{append} eq "1") {
@@ -114,10 +111,7 @@ sub isis2xml {
                        ($swish,$display) = (1,1);
 
                        while ($swish || $display) {
-                               ($swish,$display) = parse_format($format,$row,$isis_i++);
-#print STDERR "s: $swish\nd: $display\n" if ($swish);
-
-#print STDERR "swish: $swish<-- display: $display<--\n";
+                               ($swish,$display) = parse_format($format,$row,$isis_i++,$isis2cp);
 
                                # filter="name" ; filter this field through
                                # filter/[name].pm
@@ -128,7 +122,6 @@ sub isis2xml {
                                # type="swish" ; field for swish
                                if ($s && $swish) {
                                        if ($filter) {
-#print STDERR "using filter '$filter'\n";
                                                no strict 'refs';
                                                $swish_data .= join(" ",&$filter($swish));
                                        } else {
@@ -156,7 +149,7 @@ sub isis2xml {
                                                
                                # type="index" ; insert into index
                                if ($i && $display) {
-                                       my $index_data = $index_codepage->convert($display) || $display;
+                                       my $index_data = $display;
                                        if ($filter) {
                                                no strict 'refs';
                                                foreach my $d (&$filter($index_data)) {
@@ -170,13 +163,8 @@ sub isis2xml {
                }
 
 
-#print STDERR "s_d: $swish_data\nd_d: $display_data\n" if ($swish_data);
                if ($display_data) {
 
-#                      $display_data = $isis_codepage->convert($display_data) || die "Can't convert '$display_data' !";
-                       # FIX: this is removed and replaced by html tag.
-                       #$xml .= xmlify($field."_display", $display_data);
-
                        if ($field eq "headline") {
                                $xml .= xmlify("headline", $display_data);
                        } else {
@@ -193,21 +181,17 @@ sub isis2xml {
                                        print STDERR "WARNING: field '$field' doesn't have 'name' attribute!";
                                }
                                if ($field_name) {
-                                       $html .= $xml_codepage->convert($field_name);
+                                       $html .= x($field_name);
                                }
                                $html .= $display_data."###\n";
                        }
                }
                if ($swish_data) {
-                       my $i = Text::Iconv->new($config->{isis_codepage},'ISO8859-2');
                        # remove extra spaces
                        $swish_data =~ s/ +/ /g;
                        $swish_data =~ s/ +$//g;
 
-                       $swish_data = $i->convert($swish_data);
-                       $xml .= xmlify($field."_swish",unac_string('ISO8859-2',$swish_data));
-                       #$swish_data = $isis_codepage->convert($swish_data)."##" || $swish_data;
-                       #$xml .= xmlify($field."_swish",unac_string($config->{isis_codepage},$swish_data));
+                       $xml .= xmlify($field."_swish", unac_string($codepage,$swish_data));
                }
 
 
@@ -219,7 +203,6 @@ sub isis2xml {
        }
        
        if ($xml) {
-#print STDERR "x: $xml\n";
                $xml .= $add_xml if ($add_xml);
                return "<xml>\n$xml</xml>\n";
        } else {
@@ -233,15 +216,13 @@ my $cfg = new Config::IniFiles( -file => $config_file );
 
 foreach my $database ($cfg->Sections) {
 
-       my $isis_db = $cfg -> val($database, 'isis_db');
-       my $type = $cfg -> val($database, 'type');
-       my $add_xml = $cfg -> val($database, 'xml');
+       my $isis_db = $cfg -> val($database, 'isis_db') || die "$database doesn't have 'isis_db' defined!";
+       my $type = $cfg -> val($database, 'type') || die "$database doesn't have 'type' defined";
+       my $add_xml = $cfg -> val($database, 'xml');    # optional
 
-       # read configuration for this type
        $config=XMLin("./import_xml/$type.xml", forcearray => [ 'isis' ], forcecontent => 1);
-       $isis_codepage = Text::Iconv->new($config->{isis_codepage},$XML_CHARSET);
-       $index_codepage = Text::Iconv->new($config->{isis_codepage},$config->{index_codepage});
-       $xml_codepage = Text::Iconv->new($cfg->val($database,'xml_codepage'),$XML_CHARSET);
+
+       $isis2cp = Text::Iconv->new($config->{isis_codepage},$codepage);
 
        my $db = OpenIsis::open( $isis_db );
        if (0) {
@@ -264,7 +245,6 @@ foreach my $database ($cfg->Sections) {
        for (my $row_id = 1; $row_id <= $max_rowid; $row_id++ ) {
                my $row = OpenIsis::read( $db, $row_id );
                if ($row && $row->{mfn}) {
-#print STDERR "mfn: ",$row->{mfn},"\n";
                        # output current process indicator
                        my $p = int($row->{mfn} * 100 / $max_rowid);
                        if ($p != $last_p) {
@@ -272,8 +252,7 @@ foreach my $database ($cfg->Sections) {
                                $last_p = $p;
                        }
 
-                       if (my $xml = isis2xml($row,$add_xml)) {
-#print STDERR "--ret-->$xml\n";
+                       if (my $xml = $cp2utf->convert(isis2xml($row,$add_xml))) {
                                use bytes;      # as opposed to chars
                                print "Path-Name: $path#".int($row->{mfn})."\n";
                                print "Content-Length: ".(length($xml)+1)."\n";

diff --git a/import_xml/isis.xml b/import_xml/isis.xml
index cbff300..75501d4 100644 (file)
--- a/import_xml/isis.xml
+++ b/import_xml/isis.xml
@@ -7,7 +7,6 @@
 -->
 <config
        isis_codepage="852"
-       index_codepage="ISO8859-2"
 >
 
 <!--

diff --git a/parse_format.pm b/parse_format.pm
index f8dd6a3..e6f8aac 100644 (file)
--- a/parse_format.pm
+++ b/parse_format.pm
@@ -9,6 +9,7 @@ sub parse_format {
        my $format = shift;
        my $row = shift;
        my $i = shift || 0;     # isis repeatable number
+       my $codepage = shift || die;
 
        my $out;
        my $out_swish;
@@ -26,6 +27,7 @@ sub parse_format {
                if ($format =~ s/^(\d\d\d)(\w?)//) {
                        my $isis_tmp = isis_sf($row,$1,$2,$i);
                        if ($isis_tmp) {
+                               $isis_tmp = $codepage->convert($isis_tmp) if ($codepage);
                                $display .= $prefix . $isis_tmp;
                                $swish .= $isis_tmp." ";
 #print STDERR " == $isis_tmp";
@@ -37,6 +39,7 @@ sub parse_format {
                } elsif ($format =~ s/^(1[01])//) {
                        my $isis_tmp = isis_sf($row,$1,undef,$i);
                        if ($isis_tmp) {
+                               $isis_tmp = $codepage->convert($isis_tmp) if ($codepage);
                                $display .= $prefix . $isis_tmp;
                                $swish .= $isis_tmp." ";
                        }

diff --git a/test_charset.pl b/test_charset.pl
index 99fef6d..27abd73 100755 (executable)
--- a/test_charset.pl
+++ b/test_charset.pl
@@ -3,6 +3,8 @@
 use strict;
 use OpenIsis;
 use Text::Iconv;
+use XML::Simple;
+use Data::Dumper;
 
 Text::Iconv->raise_error(1);     # Conversion errors raise exceptions
 
@@ -20,5 +22,17 @@ my $row = OpenIsis::read( $db, $row_id );
 
 use isis_sf;
 
-print $isis_codepage->convert(isis_sf($row,'200')),"\n";
+print isis_sf($row,'200'),"\n";
+
+my $xml = '<?xml version="1.0" encoding="ISO-8859-2"?><xml>truæ</xml>';
+#my $xml_codepage = Text::Iconv->new('ISO-8859-2','UTF-8');
+#$xml = XMLin($xml_codepage->convert($xml));
+
+$xml = XMLin($xml);
+
+#undef $xml_codepage;
+my $xml_codepage = Text::Iconv->new('UTF-8','iso-8859-2');
+$xml = $xml_codepage->convert($xml),"\n";
+
+print Dumper($xml),"$xml\n";