From: srowen Date: Sat, 15 May 2010 11:21:03 +0000 (+0000) Subject: Move character encoding logic out to common, try again to improve its handling of... X-Git-Url: http://git.rot13.org/?p=zxing.git;a=commitdiff_plain;h=9ba879aa77a14e6a3d42b59c670937eeb64c706e Move character encoding logic out to common, try again to improve its handling of UTF8 for Chinese market, per manufacturer request, added test cases git-svn-id: http://zxing.googlecode.com/svn/trunk@1364 59b500cc-1b3d-0410-9834-0bbf25fbcc57 --- diff --git a/core/src/com/google/zxing/common/StringUtils.java b/core/src/com/google/zxing/common/StringUtils.java new file mode 100644 index 00000000..d3a7c2b6 --- /dev/null +++ b/core/src/com/google/zxing/common/StringUtils.java @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2010 ZXing authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.zxing.common; + +import java.util.Hashtable; + +import com.google.zxing.DecodeHintType; + +/** + * Common string-related functions. + * + * @author Sean Owen + */ +public final class StringUtils { + + private static final String PLATFORM_DEFAULT_ENCODING = + System.getProperty("file.encoding"); + public static final String SHIFT_JIS = "SJIS"; + private static final String EUC_JP = "EUC_JP"; + private static final String UTF8 = "UTF8"; + private static final String ISO88591 = "ISO8859_1"; + private static final boolean ASSUME_SHIFT_JIS = + SHIFT_JIS.equalsIgnoreCase(PLATFORM_DEFAULT_ENCODING) || + EUC_JP.equalsIgnoreCase(PLATFORM_DEFAULT_ENCODING); + + private StringUtils() {} + + /** + * @param bytes bytes encoding a string, whose encoding should be guessed + * @param hints decode hints if applicable + * @return name of guessed encoding; at the moment will only guess one of: + * {@link #SHIFT_JIS}, {@link #UTF8}, {@link #ISO88591}, or the platform + * default encoding if none of these can possibly be correct + */ + public static String guessEncoding(byte[] bytes, Hashtable hints) { + if (hints != null) { + String characterSet = (String) hints.get(DecodeHintType.CHARACTER_SET); + if (characterSet != null) { + return characterSet; + } + } + // Does it start with the UTF-8 byte order mark? then guess it's UTF-8 + if (bytes.length > 3 && + bytes[0] == (byte) 0xEF && + bytes[1] == (byte) 0xBB && + bytes[2] == (byte) 0xBF) { + return UTF8; + } + // For now, merely tries to distinguish ISO-8859-1, UTF-8 and Shift_JIS, + // which should be by far the most common encodings. ISO-8859-1 + // should not have bytes in the 0x80 - 0x9F range, while Shift_JIS + // uses this as a first byte of a two-byte character. If we see this + // followed by a valid second byte in Shift_JIS, assume it is Shift_JIS. + // If we see something else in that second byte, we'll make the risky guess + // that it's UTF-8. + int length = bytes.length; + boolean canBeISO88591 = true; + boolean canBeShiftJIS = true; + boolean canBeUTF8 = true; + int utf8BytesLeft = 0; + int maybeDoubleByteCount = 0; + int maybeSingleByteKatakanaCount = 0; + boolean sawLatin1Supplement = false; + boolean sawUTF8Start = false; + boolean lastWasPossibleDoubleByteStart = false; + + for (int i = 0; + i < length && (canBeISO88591 || canBeShiftJIS || canBeUTF8); + i++) { + + int value = bytes[i] & 0xFF; + + // UTF-8 stuff + if (value >= 0x80 && value <= 0xBF) { + if (utf8BytesLeft > 0) { + utf8BytesLeft--; + } + } else { + if (utf8BytesLeft > 0) { + canBeUTF8 = false; + } + if (value >= 0xC0 && value <= 0xFD) { + sawUTF8Start = true; + int valueCopy = value; + while ((valueCopy & 0x40) != 0) { + utf8BytesLeft++; + valueCopy <<= 1; + } + } + } + + // ISO-8859-1 stuff + + if ((value == 0xC2 || value == 0xC3) && i < length - 1) { + // This is really a poor hack. The slightly more exotic characters people might want to put in + // a QR Code, by which I mean the Latin-1 supplement characters (e.g. u-umlaut) have encodings + // that start with 0xC2 followed by [0xA0,0xBF], or start with 0xC3 followed by [0x80,0xBF]. + int nextValue = bytes[i + 1] & 0xFF; + if (nextValue <= 0xBF && + ((value == 0xC2 && nextValue >= 0xA0) || (value == 0xC3 && nextValue >= 0x80))) { + sawLatin1Supplement = true; + } + } + if (value >= 0x7F && value <= 0x9F) { + canBeISO88591 = false; + } + + // Shift_JIS stuff + + if (value >= 0xA1 && value <= 0xDF) { + // count the number of characters that might be a Shift_JIS single-byte Katakana character + if (!lastWasPossibleDoubleByteStart) { + maybeSingleByteKatakanaCount++; + } + } + if (!lastWasPossibleDoubleByteStart && + ((value >= 0xF0 && value <= 0xFF) || value == 0x80 || value == 0xA0)) { + canBeShiftJIS = false; + } + if (((value >= 0x81 && value <= 0x9F) || (value >= 0xE0 && value <= 0xEF))) { + // These start double-byte characters in Shift_JIS. Let's see if it's followed by a valid + // second byte. + if (lastWasPossibleDoubleByteStart) { + // If we just checked this and the last byte for being a valid double-byte + // char, don't check starting on this byte. If this and the last byte + // formed a valid pair, then this shouldn't be checked to see if it starts + // a double byte pair of course. + lastWasPossibleDoubleByteStart = false; + } else { + // ... otherwise do check to see if this plus the next byte form a valid + // double byte pair encoding a character. + lastWasPossibleDoubleByteStart = true; + if (i >= bytes.length - 1) { + canBeShiftJIS = false; + } else { + int nextValue = bytes[i + 1] & 0xFF; + if (nextValue < 0x40 || nextValue > 0xFC) { + canBeShiftJIS = false; + } else { + maybeDoubleByteCount++; + } + // There is some conflicting information out there about which bytes can follow which in + // double-byte Shift_JIS characters. The rule above seems to be the one that matches practice. + } + } + } else { + lastWasPossibleDoubleByteStart = false; + } + } + if (utf8BytesLeft > 0) { + canBeUTF8 = false; + } + + // Easy -- if assuming Shift_JIS and no evidence it can't be, done + if (canBeShiftJIS && ASSUME_SHIFT_JIS) { + return SHIFT_JIS; + } + if (canBeUTF8 && sawUTF8Start) { + return UTF8; + } + // Distinguishing Shift_JIS and ISO-8859-1 can be a little tough. The crude heuristic is: + // - If we saw + // - at least 3 bytes that starts a double-byte value (bytes that are rare in ISO-8859-1), or + // - over 5% of bytes could be single-byte Katakana (also rare in ISO-8859-1), + // - and, saw no sequences that are invalid in Shift_JIS, then we conclude Shift_JIS + if (canBeShiftJIS && (maybeDoubleByteCount >= 3 || 20 * maybeSingleByteKatakanaCount > length)) { + return SHIFT_JIS; + } + // Otherwise, we default to ISO-8859-1 unless we know it can't be + if (!sawLatin1Supplement && canBeISO88591) { + return ISO88591; + } + // Otherwise, we take a wild guess with platform encoding + return PLATFORM_DEFAULT_ENCODING; + } + +} diff --git a/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java b/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java index 25907e3b..94325d3d 100644 --- a/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java +++ b/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java @@ -16,11 +16,11 @@ package com.google.zxing.qrcode.decoder; -import com.google.zxing.DecodeHintType; import com.google.zxing.FormatException; import com.google.zxing.common.BitSource; import com.google.zxing.common.CharacterSetECI; import com.google.zxing.common.DecoderResult; +import com.google.zxing.common.StringUtils; import java.io.UnsupportedEncodingException; import java.util.Hashtable; @@ -45,16 +45,6 @@ final class DecodedBitStreamParser { 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', ' ', '$', '%', '*', '+', '-', '.', '/', ':' }; - private static final String SHIFT_JIS = "SJIS"; - private static final String EUC_JP = "EUC_JP"; - private static final boolean ASSUME_SHIFT_JIS; - private static final String UTF8 = "UTF8"; - private static final String ISO88591 = "ISO8859_1"; - - static { - String platformDefault = System.getProperty("file.encoding"); - ASSUME_SHIFT_JIS = SHIFT_JIS.equalsIgnoreCase(platformDefault) || EUC_JP.equalsIgnoreCase(platformDefault); - } private DecodedBitStreamParser() { } @@ -140,7 +130,7 @@ final class DecodedBitStreamParser { } // Shift_JIS may not be supported in some environments: try { - result.append(new String(buffer, SHIFT_JIS)); + result.append(new String(buffer, StringUtils.SHIFT_JIS)); } catch (UnsupportedEncodingException uee) { throw FormatException.getFormatInstance(); } @@ -166,7 +156,7 @@ final class DecodedBitStreamParser { // upon decoding. I have seen ISO-8859-1 used as well as // Shift_JIS -- without anything like an ECI designator to // give a hint. - encoding = guessEncoding(readBytes, hints); + encoding = StringUtils.guessEncoding(readBytes, hints); } else { encoding = currentCharacterSetECI.getEncodingName(); } @@ -243,103 +233,6 @@ final class DecodedBitStreamParser { result.append(ALPHANUMERIC_CHARS[digitBits]); } } - - private static String guessEncoding(byte[] bytes, Hashtable hints) { - if (hints != null) { - String characterSet = (String) hints.get(DecodeHintType.CHARACTER_SET); - if (characterSet != null) { - return characterSet; - } - } - if (ASSUME_SHIFT_JIS) { - return SHIFT_JIS; - } - // Does it start with the UTF-8 byte order mark? then guess it's UTF-8 - if (bytes.length > 3 && bytes[0] == (byte) 0xEF && bytes[1] == (byte) 0xBB && bytes[2] == (byte) 0xBF) { - return UTF8; - } - // For now, merely tries to distinguish ISO-8859-1, UTF-8 and Shift_JIS, - // which should be by far the most common encodings. ISO-8859-1 - // should not have bytes in the 0x80 - 0x9F range, while Shift_JIS - // uses this as a first byte of a two-byte character. If we see this - // followed by a valid second byte in Shift_JIS, assume it is Shift_JIS. - // If we see something else in that second byte, we'll make the risky guess - // that it's UTF-8. - int length = bytes.length; - boolean canBeISO88591 = true; - boolean canBeShiftJIS = true; - int maybeDoubleByteCount = 0; - int maybeSingleByteKatakanaCount = 0; - boolean sawLatin1Supplement = false; - boolean lastWasPossibleDoubleByteStart = false; - for (int i = 0; i < length && (canBeISO88591 || canBeShiftJIS); i++) { - int value = bytes[i] & 0xFF; - if ((value == 0xC2 || value == 0xC3) && i < length - 1) { - // This is really a poor hack. The slightly more exotic characters people might want to put in - // a QR Code, by which I mean the Latin-1 supplement characters (e.g. u-umlaut) have encodings - // that start with 0xC2 followed by [0xA0,0xBF], or start with 0xC3 followed by [0x80,0xBF]. - int nextValue = bytes[i + 1] & 0xFF; - if (nextValue <= 0xBF && ((value == 0xC2 && nextValue >= 0xA0) || (value == 0xC3 && nextValue >= 0x80))) { - sawLatin1Supplement = true; - } - } - if (value >= 0x7F && value <= 0x9F) { - canBeISO88591 = false; - } - if (value >= 0xA1 && value <= 0xDF) { - // count the number of characters that might be a Shift_JIS single-byte Katakana character - if (!lastWasPossibleDoubleByteStart) { - maybeSingleByteKatakanaCount++; - } - } - if (!lastWasPossibleDoubleByteStart && ((value >= 0xF0 && value <= 0xFF) || value == 0x80 || value == 0xA0)) { - canBeShiftJIS = false; - } - if (((value >= 0x81 && value <= 0x9F) || (value >= 0xE0 && value <= 0xEF))) { - // These start double-byte characters in Shift_JIS. Let's see if it's followed by a valid - // second byte. - if (lastWasPossibleDoubleByteStart) { - // If we just checked this and the last byte for being a valid double-byte - // char, don't check starting on this byte. If this and the last byte - // formed a valid pair, then this shouldn't be checked to see if it starts - // a double byte pair of course. - lastWasPossibleDoubleByteStart = false; - } else { - // ... otherwise do check to see if this plus the next byte form a valid - // double byte pair encoding a character. - lastWasPossibleDoubleByteStart = true; - if (i >= bytes.length - 1) { - canBeShiftJIS = false; - } else { - int nextValue = bytes[i + 1] & 0xFF; - if (nextValue < 0x40 || nextValue > 0xFC) { - canBeShiftJIS = false; - } else { - maybeDoubleByteCount++; - } - // There is some conflicting information out there about which bytes can follow which in - // double-byte Shift_JIS characters. The rule above seems to be the one that matches practice. - } - } - } else { - lastWasPossibleDoubleByteStart = false; - } - } - // Distinguishing Shift_JIS and ISO-8859-1 can be a little tough. The crude heuristic is: - // - If we saw - // - at least three byte that starts a double-byte value (bytes that are rare in ISO-8859-1), or - // - over 5% of bytes that could be single-byte Katakana (also rare in ISO-8859-1), - // - and, saw no sequences that are invalid in Shift_JIS, then we conclude Shift_JIS - if (canBeShiftJIS && (maybeDoubleByteCount >= 3 || 20 * maybeSingleByteKatakanaCount > length)) { - return SHIFT_JIS; - } - // Otherwise, we default to ISO-8859-1 unless we know it can't be - if (!sawLatin1Supplement && canBeISO88591) { - return ISO88591; - } - // Otherwise, we take a wild guess with UTF-8 - return UTF8; - } private static int parseECIValue(BitSource bits) { int firstByte = bits.readBits(8); diff --git a/core/test/data/blackbox/qrcode-2/29.jpg b/core/test/data/blackbox/qrcode-2/29.jpg new file mode 100644 index 00000000..3b4dc691 Binary files /dev/null and b/core/test/data/blackbox/qrcode-2/29.jpg differ diff --git a/core/test/data/blackbox/qrcode-2/29.txt b/core/test/data/blackbox/qrcode-2/29.txt new file mode 100644 index 00000000..87ad2998 --- /dev/null +++ b/core/test/data/blackbox/qrcode-2/29.txt @@ -0,0 +1,3 @@ +http://live.fdgm.jp/u/event/hype/hype_top.html + +MEBKM:TITLE:hypeモバイル;URL:http\://live.fdgm.jp/u/event/hype/hype_top.html;; \ No newline at end of file diff --git a/core/test/data/blackbox/qrcode-2/30.png b/core/test/data/blackbox/qrcode-2/30.png new file mode 100644 index 00000000..f6407bd6 Binary files /dev/null and b/core/test/data/blackbox/qrcode-2/30.png differ diff --git a/core/test/data/blackbox/qrcode-2/30.txt b/core/test/data/blackbox/qrcode-2/30.txt new file mode 100644 index 00000000..bc64130b --- /dev/null +++ b/core/test/data/blackbox/qrcode-2/30.txt @@ -0,0 +1 @@ +MECARD:N:測試;; \ No newline at end of file diff --git a/core/test/data/blackbox/qrcode-2/31.jpg b/core/test/data/blackbox/qrcode-2/31.jpg new file mode 100644 index 00000000..9c63c0cc Binary files /dev/null and b/core/test/data/blackbox/qrcode-2/31.jpg differ diff --git a/core/test/data/blackbox/qrcode-2/31.txt b/core/test/data/blackbox/qrcode-2/31.txt new file mode 100644 index 00000000..59d2e9da --- /dev/null +++ b/core/test/data/blackbox/qrcode-2/31.txt @@ -0,0 +1 @@ +今度のバージョンでは文章の暗号化ができます。 \ No newline at end of file diff --git a/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java b/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java index b31caa4e..80bac2f7 100644 --- a/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java +++ b/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java @@ -27,10 +27,10 @@ public final class QRCodeBlackBox2TestCase extends AbstractBlackBoxTestCase { public QRCodeBlackBox2TestCase() { super("test/data/blackbox/qrcode-2", new MultiFormatReader(), BarcodeFormat.QR_CODE); - addTest(21, 21, 0.0f); - addTest(18, 18, 90.0f); - addTest(20, 20, 180.0f); - addTest(18, 18, 270.0f); + addTest(23, 23, 0.0f); + addTest(21, 21, 90.0f); + addTest(23, 23, 180.0f); + addTest(20, 21, 270.0f); } } diff --git a/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java b/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java index ddd1052e..eabb134c 100644 --- a/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java +++ b/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java @@ -41,13 +41,14 @@ public final class DecodedBitStreamParserTestCase extends TestCase { public void testSimpleSJIS() throws Exception { BitSourceBuilder builder = new BitSourceBuilder(); builder.write(0x04, 4); // Byte mode - builder.write(0x03, 8); // 3 bytes + builder.write(0x04, 8); // 4 bytes builder.write(0xA1, 8); builder.write(0xA2, 8); builder.write(0xA3, 8); + builder.write(0xD0, 8); String result = DecodedBitStreamParser.decode(builder.toByteArray(), Version.getVersionForNumber(1), null, null).getText(); - assertEquals("\uff61\uff62\uff63", result); + assertEquals("\uff61\uff62\uff63\uff90", result); } public void testECI() throws Exception {