X-Git-Url: http://git.rot13.org/?a=blobdiff_plain;f=lib%2FMojoFacets%2FImport%2FCSV.pm;h=e8fa19624cc809f514e3438226b3d8dde7f63d3f;hb=ce2e4a650a6a3c52ddccc2382488c6ff4a2ed763;hp=6ab33951858e6cb48f520fca6d52ee798e88a2d9;hpb=f22128eabec21566ad71187c22df6439e4881bcd;p=MojoFacets.git

diff --git a/lib/MojoFacets/Import/CSV.pm b/lib/MojoFacets/Import/CSV.pm
index 6ab3395..e8fa196 100644
--- a/lib/MojoFacets/Import/CSV.pm
+++ b/lib/MojoFacets/Import/CSV.pm
@@ -5,90 +5,74 @@ use strict;
 
 use base 'Mojo::Base';
 
-use File::Slurp;
+use Text::CSV;
 use Data::Dump qw(dump);
-use Encode;
 
 __PACKAGE__->attr('full_path');
 
-my $null = ''; # FIXME undef?
-
-sub _split_line {
-	my ( $delimiter, $line ) = @_;
-	my @v;
-	while ( $line ) {
-		my $v;
-		if ( $line =~ s/^"// ) {
-			$line =~ s/""/_qq_/gc;
-			$line =~ s/^\s*([^"]*)\s*"\Q$delimiter\E?// || die "can't parse [$line] ",dump(@v);
-			$v = $1;
-		} elsif ( $line =~ s/^\s*([^\Q$delimiter\E]+)\s*\Q$delimiter\E?// ) {
-			$v = $1;
-		} elsif ( $line =~ s/^\s*\Q$delimiter\E// ) {
-			$v = $null;
-		} else {
-			die "can't parse [$line]\n";
-		}
-
-		$v =~ s/^\s*(.+?)\s*$/$1/;
-		$v = $null if $v eq '_qq_'; # "" field which is not first one
-		$v =~ s/_qq_/"/g;
-		$v =~ s/_LF_/\n/g;
-		push @v, $v;
-	}
-
-	return @v;
-}
+sub ext { '\.[ct]sv$' };
 
 sub data {
 	my $self = shift;
 
 	my $path = $self->full_path;
 
-	my $data = read_file $path, { binmode => ':raw' }; # FIXME configurable!
 	my $encoding = 'utf-8';
-	if ( $path =~ m/\.(\w+).csv/i ) {
+	if ( $path =~ m/\.([\w\-]+).[ct]sv/i ) {
 		$encoding = $1;
 	}
-	warn "decoding ", length($data), " bytes using $encoding\n";
-	$data = decode($encoding, $data);
-
-	# multi-line strings
-	while ( $data =~ s/(,"[^"]*)[\n\r]+([^"]*)/$1_LF_$2/sg ) {
-		 warn "multi-line quoted CSV data found";
-	}
 
-	my @lines = split(/\r?\n/, $data);
-	$data = { items => [] };
+	my $data = { items => [] };
+	my @header;
 
-	my $delimiter = ',';
+	open my $fh, "<:encoding($encoding)", $path or die "$path: $!";
+	my $first = <$fh>;
+	my $possible_delimiters;
+	while ( $first =~ s/(\W)// ) {
+		$possible_delimiters->{$1}++;
+	}
+	warn "# possible_delimiters = ",dump($possible_delimiters);
+	seek $fh,0,0; # rewind for Text::CSV
+
+	my @sep_by_usage = sort { $possible_delimiters->{$b} <=> $possible_delimiters->{$a} } keys %$possible_delimiters;
+	my $sep_char = shift @sep_by_usage;
+	while ( $sep_char =~ m/^\s$/ ) {
+		last if $sep_char eq "\t" && $path =~ m/\.tsv$/i;
+		warn "## skip whitespace separator ",dump($sep_char);
+		$sep_char = shift @sep_by_usage;
+	}
 
-	if ( $lines[0] !~ m/,/ ) {
-		if ( $lines[0] =~ m/;/ ) {
-			$delimiter = ';';
-		} elsif ( $lines[0] !~ /;/ && $lines[1] =~ /;/ ) {
-			shift @lines; # FIXME skip non-header line
-			$delimiter = ';';
-		}
+	while ( $sep_char =~ m/^\"$/ ) {
+		warn "## skip quote separator ",dump($sep_char);
+		$sep_char = shift @sep_by_usage;
 	}
 
-	warn "$path ", $#lines + 1, " lines encoding: $encoding delimiter:",dump($delimiter);
+	if ( $sep_char !~ m/,/ && $possible_delimiters->{','} && $path =~ m/\.csv/i ) {
+		$sep_char = ',';
+		warn "## csv file detected so prefer , as separator";
+	}
 
-	my $header_line = shift @lines;
+	warn "sep_char = [$sep_char] for $path\n";
 
-	my @header = _split_line( $delimiter, $header_line );
-	warn "# header ",dump( @header );
+	my $csv = Text::CSV->new ( { binary => 1, eol => $/, sep_char => $sep_char } )
+		or die "Cannot use CSV: ".Text::CSV->error_diag ();
 
-	while ( my $line = shift @lines ) {
-		chomp $line;
-		my @v = _split_line($delimiter, $line);
+	while ( my $row = $csv->getline( $fh ) ) {
+		if ( ! @header ) {
+			@header = @$row;
+			$header[0] =~ s/^#// if $path =~ m/\.tsv/i; # remove hash from 1st column
+			next;
+		}
 		my $item;
-		foreach my $i ( 0 .. $#v ) {
-			$item->{ $header[$i] || "f_$i" } = [ $v[$i] ];
+		foreach my $i ( 0 .. $#{$row} ) {
+			$item->{ $header[$i] || "f_$i" } = [ $row->[$i] ];
 		}
 		push @{ $data->{items} }, $item;
 	}
 
+	$csv->eof or $csv->error_diag();
+	close $fh;
+
 	$data->{header} = [ @header ];
 	
 	return $data;