- my($it, $kind);
- my $eof_p = 0;
- if (!defined $readahead || !length $readahead) {
- my $next = scalar <$h>;
- $eof_p = !defined $next;
- if (!$eof_p) {
- $lc += 1;
- $readahead .= $next;
- }
- }
- $lc_0 = $lc; # remember line number of first line
- if ($eof_p && !length $readahead) { # nothing left to do
- ;
- } elsif ($readahead =~ /^\s+/s) { # whitespace
- ($kind, $it, $readahead) = (KIND_TEXT, $&, $');
- # FIXME the following (the [<\s] part) is an unreliable HACK :-(
- } elsif ($readahead =~ /^(?:[^<]|<[<\s])+/s) { # non-space normal text
- ($kind, $it, $readahead) = (KIND_TEXT, $&, $');
- warn "Warning: Unescaped < in line $lc: $it\n" if $it =~ /</s;
- } else { # tag/declaration/processing instruction
- my $ok_p = 0;
- for (;;) {
- if ($cdata_mode_p) {
- if ($readahead =~ /^$cdata_close/) {
- ($kind, $it, $readahead) = (KIND_TAG, $&, $');
- $ok_p = 1;
- } else {
- ($kind, $it, $readahead) = (KIND_TEXT, $readahead, undef);
- $ok_p = 1;
- }
- } elsif ($readahead =~ /^$re_tag_compat/os) {
- ($kind, $it, $readahead) = (KIND_TAG, "$1$2", $3);
- $ok_p = 1;
- } elsif ($readahead =~ /^<!--(?:(?!-->).)*-->/s) {
- ($kind, $it, $readahead) = (KIND_COMMENT, $&, $');
- $ok_p = 1;
- warn "Warning: Syntax error in comment at line $lc_0: $&\n";
- $syntaxerror_p = 1;
- }
- last if $ok_p;
- my $next = scalar <$h>;
- $eof_p = !defined $next;
- last if $eof_p;
- $lc += 1;
- $readahead .= $next;
- }
- if ($kind ne KIND_TAG) {
- ;
- } elsif ($it =~ /^<!/) {
- $kind = KIND_DECL;
- $kind = KIND_COMMENT if $it =~ /^<!--(?:(?!-->).)*-->/;
- } elsif ($it =~ /^<\?/) {
- $kind = KIND_PI;
- }
- if ($it =~ /^$re_directive/ios && !$cdata_mode_p) {
- $kind = KIND_DIRECTIVE;
- }
- if (!$ok_p && $eof_p) {
- ($kind, $it, $readahead) = (KIND_UNKNOWN, $readahead, undef);
- $syntaxerror_p = 1;
- }
- }
- warn "Warning: Unrecognizable token found in line $lc_0: $it\n"
- if $kind eq KIND_UNKNOWN;
- return defined $it? (wantarray? ($kind, $it):
- [$kind, $it]): undef;
-}
-
-sub next_token (*) {
- my($h) = @_;
- my $it;
- if (!$cdata_mode_p) {
- $it = next_token_internal($h);
- if (defined $it && $it->[0] eq KIND_TAG) { # FIXME
- ($cdata_mode_p, $cdata_close) = (1, "</$1\\s*>")
- if $it->[1] =~ /^<(script|style|textarea)\b/i; #FIXME
- push @$it, extract_attributes($it->[1], $lc); #FIXME
- }
- } else {
- for (;;) {
- my $lc_prev = $lc;
- my $next = next_token_internal($h);
- last if !defined $next;
- if (defined $next && $next->[1] =~ /$cdata_close/i) { #FIXME
- ($lc, $readahead) = ($lc_prev, $next->[1] . $readahead); #FIXME
- $cdata_mode_p = 0;
- }
- last unless $cdata_mode_p;
- $it .= $next->[1]; #FIXME
- }
- $it = [KIND_CDATA, $it] if defined $it; #FIXME
- $cdata_close = undef;
- }
- return defined $it? (wantarray? @$it: $it): undef;
-}
-
-###############################################################################
-
-sub debug_dump (*) { # for testing only
- my($h) = @_;
- print "re_tag_compat is /$re_tag_compat/\n";