rewrite CSV parser to support "quoted" fiends

[MojoFacets.git] / lib / MojoFacets / Import / CSV.pm

diff --git a/lib/MojoFacets/Import/CSV.pm b/lib/MojoFacets/Import/CSV.pm
index 41c168a..ae8a518 100644 (file)
--- a/lib/MojoFacets/Import/CSV.pm
+++ b/lib/MojoFacets/Import/CSV.pm
@@ -5,35 +5,66 @@ use strict;
 
 use base 'Mojo::Base';
 
-use HTML::TableExtract;
 use File::Slurp;
 use Data::Dump qw(dump);
-use JSON;
+use Encode;
 
 __PACKAGE__->attr('path');
-__PACKAGE__->attr('full_path');
+__PACKAGE__->attr('full_path'); # FIXME remove full_path
+
+my $null = ''; # FIXME undef?
+
+sub _split_line {
+       my ( $delimiter, $line ) = @_;
+       my @v;
+       while ( $line ) {
+               if ( $line =~ s/^"([^"]+)"\Q$delimiter\E?// ) {
+                       push @v, $1;
+               } elsif ( $line =~ s/^([^\Q$delimiter\E]+)\Q$delimiter\E?// ) {
+                       push @v, $1;
+               } elsif ( $line =~ s/^\Q$delimiter\E// ) {
+                       push @v, $null;
+               } else {
+                       die "can't parse [$line]\n";
+               }
+       }
+
+       return @v;
+}
 
 sub data {
        my $self = shift;
 
-       my $path = $self->path;
+       my $path = $self->full_path || $self->path;
 
-       my $data = read_file $self->full_path, { binmode => ':cp1250' }; # FIXME configurable!
+       my $data = read_file $path, { binmode => ':raw' }; # FIXME configurable!
+       my $encoding = 'utf-8';
+       if ( $path =~ m/\.(\w+).csv/i ) {
+               $encoding = $1;
+       }
+       warn "decoding ", length($data), " bytes using $encoding\n";
+       $data = decode($encoding, $data);
 
        my @lines = split(/\r?\n/, $data);
        $data = { items => [] };
 
-       my $delimiter = qr/;/;
+       my $delimiter = ',';
+
+       if ( $lines[0] !~ /;/ && $lines[1] =~ /;/ ) {
+               shift @lines; # FIXME ship non-header line
+               $delimiter = ';';
+       }
+
+       warn "$path ", $#lines + 1, " lines encoding: $encoding delimiter:",dump($delimiter);
 
-       shift @lines; # FIXME ship non-header line
        my $header_line = shift @lines;
 
-       my @header = split( $delimiter, $header_line );
+       my @header = _split_line( $delimiter, $header_line );
        warn "# header ",dump( @header );
 
        while ( my $line = shift @lines ) {
                chomp $line;
-               my @v = split($delimiter, $line);
+               my @v = _split_line($delimiter, $line);
                my $item;
                foreach my $i ( 0 .. $#v ) {
                        $item->{ $header[$i] || "f_$i" } = [ $v[$i] ];