-
- private static String guessEncoding(byte[] bytes) {
- if (ASSUME_SHIFT_JIS) {
- return SHIFT_JIS;
- }
- // Does it start with the UTF-8 byte order mark? then guess it's UTF-8
- if (bytes.length > 3 && bytes[0] == (byte) 0xEF && bytes[1] == (byte) 0xBB && bytes[2] == (byte) 0xBF) {
- return UTF8;
- }
- // For now, merely tries to distinguish ISO-8859-1, UTF-8 and Shift_JIS,
- // which should be by far the most common encodings. ISO-8859-1
- // should not have bytes in the 0x80 - 0x9F range, while Shift_JIS
- // uses this as a first byte of a two-byte character. If we see this
- // followed by a valid second byte in Shift_JIS, assume it is Shift_JIS.
- // If we see something else in that second byte, we'll make the risky guess
- // that it's UTF-8.
- int length = bytes.length;
- boolean canBeISO88591 = true;
- boolean canBeShiftJIS = true;
- boolean sawDoubleByteStart = false;
- int maybeSingleByteKatakanaCount = 0;
- boolean lastWasPossibleDoubleByteStart = false;
- for (int i = 0; i < length && (canBeISO88591 || canBeShiftJIS); i++) {
- int value = bytes[i] & 0xFF;
- if (value >= 0x7F && value <= 0x9F) {
- canBeISO88591 = false;
- }
- if (value >= 0xA1 && value <= 0xDF) {
- // count the number of characters that might be a Shift_JIS single-byte Katakana character
- if (!lastWasPossibleDoubleByteStart) {
- maybeSingleByteKatakanaCount++;
- }
- }
- if (!lastWasPossibleDoubleByteStart && ((value >= 0xF0 && value <= 0xFF) || value == 0x80 || value == 0xA0)) {
- canBeShiftJIS = false;
- }
- if (((value >= 0x81 && value <= 0x9F) || (value >= 0xE0 && value <= 0xEF)) && i < length - 1) {
- // These start double-byte characters in Shift_JIS. Let's see if it's followed by a valid
- // second byte.
- sawDoubleByteStart = true;
- if (lastWasPossibleDoubleByteStart) {
- // If we just checked this and the last byte for being a valid double-byte
- // char, don't check starting on this byte. If this and the last byte
- // formed a valid pair, then this shouldn't be checked to see if it starts
- // a double byte pair of course.
- lastWasPossibleDoubleByteStart = false;
- } else {
- // ... otherwise do check to see if this plus the next byte form a valid
- // double byte pair encoding a character.
- lastWasPossibleDoubleByteStart = true;
- int nextValue = bytes[i + 1] & 0xFF;
- if (nextValue < 0x40 || nextValue > 0xFC) {
- canBeShiftJIS = false;
- }
- // There is some conflicting information out there about which bytes can follow which in
- // double-byte Shift_JIS characters. The rule above seems to be the one that matches practice.
- }
- } else {
- lastWasPossibleDoubleByteStart = false;
- }
- }
- // Distinguishing Shift_JIS and ISO-8859-1 can be a little tough. The crude heuristic is:
- // - If we saw
- // - at least one byte that starts a double-byte value (bytes that are rare in ISO-8859-1), or
- // - over 5% of bytes that could be single-byte Katakana (also rare in ISO-8859-1),
- // - and, saw no sequences that are invalid in Shift_JIS, then we conclude Shift_JIS
- if ((sawDoubleByteStart || 20 * maybeSingleByteKatakanaCount > length) && canBeShiftJIS) {
- return SHIFT_JIS;
- }
- // Otherwise, we default to ISO-8859-1 unless we know it can't be
- if (canBeISO88591) {
- return ISO88591;
- }
- // Otherwise, we take a wild guess with UTF-8
- return UTF8;
- }