+ my $data = read_file $full_path;
+ warn "# data snippet: ", substr($data,0,200);
+ my @header;
+ if ( $path =~ m/\.js/ ) {
+ Encode::_utf8_on($data);
+ $data = from_json $data;
+ } elsif ( $path =~ m/\.txt/ ) {
+ my @lines = split(/\r?\n/, $data);
+ $data = { items => [] };
+
+ my $header_line = shift @lines;
+ my $multiline = $header_line =~ s/\^//g;
+ @header = split(/\|/, $header_line );
+ warn "# header ", dump( @header );
+ while ( my $line = shift @lines ) {
+ $line =~ s/\^//g;
+ chomp $line;
+ my @v = split(/\|/, $line);
+ while ( @lines && $#v < $#header ) {
+ $line = $lines[0];
+ $line =~ s/\^//g;
+ chomp $line;
+ my @more_v = split(/\|/, $line);
+ if ( $#v + $#more_v > $#header ) {
+ warn "short line: ",dump( @v );
+ last;
+ }
+ shift @lines;
+ $v[ $#v ] .= shift @more_v if @more_v;
+ push @v, @more_v if @more_v;
+
+ if ( $#v > $#header ) {
+ die "# splice $#header ", dump( @v );
+ @v = splice @v, 0, $#header;
+ }
+ }
+ my $item;
+ foreach my $i ( 0 .. $#v ) {
+ my $v = $v[$i];
+ # unpack numeric values separated by commas
+ my $a = $v =~ m/\d+\s*,\s*\d+/ ? [ split(/\,\s*/,$v) ] : [ $v ];
+ $item->{ $header[$i] || "f_$i" } = $a;
+ }
+ push @{ $data->{items} }, $item;
+ }
+ } else {
+ warn "file format unknown $path";
+ }