2 * Copyright (C) 2010 ZXing authors
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package com.google.zxing.common;
19 import java.util.Hashtable;
21 import com.google.zxing.DecodeHintType;
24 * Common string-related functions.
28 public final class StringUtils {
30 private static final String PLATFORM_DEFAULT_ENCODING =
31 System.getProperty("file.encoding");
32 public static final String SHIFT_JIS = "SJIS";
33 private static final String EUC_JP = "EUC_JP";
34 private static final String UTF8 = "UTF8";
35 private static final String ISO88591 = "ISO8859_1";
36 private static final boolean ASSUME_SHIFT_JIS =
37 SHIFT_JIS.equalsIgnoreCase(PLATFORM_DEFAULT_ENCODING) ||
38 EUC_JP.equalsIgnoreCase(PLATFORM_DEFAULT_ENCODING);
40 private StringUtils() {}
43 * @param bytes bytes encoding a string, whose encoding should be guessed
44 * @param hints decode hints if applicable
45 * @return name of guessed encoding; at the moment will only guess one of:
46 * {@link #SHIFT_JIS}, {@link #UTF8}, {@link #ISO88591}, or the platform
47 * default encoding if none of these can possibly be correct
49 public static String guessEncoding(byte[] bytes, Hashtable hints) {
51 String characterSet = (String) hints.get(DecodeHintType.CHARACTER_SET);
52 if (characterSet != null) {
56 // Does it start with the UTF-8 byte order mark? then guess it's UTF-8
57 if (bytes.length > 3 &&
58 bytes[0] == (byte) 0xEF &&
59 bytes[1] == (byte) 0xBB &&
60 bytes[2] == (byte) 0xBF) {
63 // For now, merely tries to distinguish ISO-8859-1, UTF-8 and Shift_JIS,
64 // which should be by far the most common encodings. ISO-8859-1
65 // should not have bytes in the 0x80 - 0x9F range, while Shift_JIS
66 // uses this as a first byte of a two-byte character. If we see this
67 // followed by a valid second byte in Shift_JIS, assume it is Shift_JIS.
68 // If we see something else in that second byte, we'll make the risky guess
70 int length = bytes.length;
71 boolean canBeISO88591 = true;
72 boolean canBeShiftJIS = true;
73 boolean canBeUTF8 = true;
74 int utf8BytesLeft = 0;
75 int maybeDoubleByteCount = 0;
76 int maybeSingleByteKatakanaCount = 0;
77 boolean sawLatin1Supplement = false;
78 boolean sawUTF8Start = false;
79 boolean lastWasPossibleDoubleByteStart = false;
82 i < length && (canBeISO88591 || canBeShiftJIS || canBeUTF8);
85 int value = bytes[i] & 0xFF;
88 if (value >= 0x80 && value <= 0xBF) {
89 if (utf8BytesLeft > 0) {
93 if (utf8BytesLeft > 0) {
96 if (value >= 0xC0 && value <= 0xFD) {
98 int valueCopy = value;
99 while ((valueCopy & 0x40) != 0) {
108 if ((value == 0xC2 || value == 0xC3) && i < length - 1) {
109 // This is really a poor hack. The slightly more exotic characters people might want to put in
110 // a QR Code, by which I mean the Latin-1 supplement characters (e.g. u-umlaut) have encodings
111 // that start with 0xC2 followed by [0xA0,0xBF], or start with 0xC3 followed by [0x80,0xBF].
112 int nextValue = bytes[i + 1] & 0xFF;
113 if (nextValue <= 0xBF &&
114 ((value == 0xC2 && nextValue >= 0xA0) || (value == 0xC3 && nextValue >= 0x80))) {
115 sawLatin1Supplement = true;
118 if (value >= 0x7F && value <= 0x9F) {
119 canBeISO88591 = false;
124 if (value >= 0xA1 && value <= 0xDF) {
125 // count the number of characters that might be a Shift_JIS single-byte Katakana character
126 if (!lastWasPossibleDoubleByteStart) {
127 maybeSingleByteKatakanaCount++;
130 if (!lastWasPossibleDoubleByteStart &&
131 ((value >= 0xF0 && value <= 0xFF) || value == 0x80 || value == 0xA0)) {
132 canBeShiftJIS = false;
134 if (((value >= 0x81 && value <= 0x9F) || (value >= 0xE0 && value <= 0xEF))) {
135 // These start double-byte characters in Shift_JIS. Let's see if it's followed by a valid
137 if (lastWasPossibleDoubleByteStart) {
138 // If we just checked this and the last byte for being a valid double-byte
139 // char, don't check starting on this byte. If this and the last byte
140 // formed a valid pair, then this shouldn't be checked to see if it starts
141 // a double byte pair of course.
142 lastWasPossibleDoubleByteStart = false;
144 // ... otherwise do check to see if this plus the next byte form a valid
145 // double byte pair encoding a character.
146 lastWasPossibleDoubleByteStart = true;
147 if (i >= bytes.length - 1) {
148 canBeShiftJIS = false;
150 int nextValue = bytes[i + 1] & 0xFF;
151 if (nextValue < 0x40 || nextValue > 0xFC) {
152 canBeShiftJIS = false;
154 maybeDoubleByteCount++;
156 // There is some conflicting information out there about which bytes can follow which in
157 // double-byte Shift_JIS characters. The rule above seems to be the one that matches practice.
161 lastWasPossibleDoubleByteStart = false;
164 if (utf8BytesLeft > 0) {
168 // Easy -- if assuming Shift_JIS and no evidence it can't be, done
169 if (canBeShiftJIS && ASSUME_SHIFT_JIS) {
172 if (canBeUTF8 && sawUTF8Start) {
175 // Distinguishing Shift_JIS and ISO-8859-1 can be a little tough. The crude heuristic is:
177 // - at least 3 bytes that starts a double-byte value (bytes that are rare in ISO-8859-1), or
178 // - over 5% of bytes could be single-byte Katakana (also rare in ISO-8859-1),
179 // - and, saw no sequences that are invalid in Shift_JIS, then we conclude Shift_JIS
180 if (canBeShiftJIS && (maybeDoubleByteCount >= 3 || 20 * maybeSingleByteKatakanaCount > length)) {
183 // Otherwise, we default to ISO-8859-1 unless we know it can't be
184 if (!sawLatin1Supplement && canBeISO88591) {
187 // Otherwise, we take a wild guess with platform encoding
188 return PLATFORM_DEFAULT_ENCODING;