Revamp the character encoding detection to use a more complicated but still crude...

git-svn-id: http://zxing.googlecode.com/svn/trunk@664 59b500cc-1b3d-0410-9834-0bbf25fbcc57

diff --git a/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java b/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java
index 287e56c..f948980 100644 (file)
--- a/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java
+++ b/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java
@@ -254,14 +254,23 @@ final class DecodedBitStreamParser {
     // that it's UTF-8.
     int length = bytes.length;
     boolean canBeISO88591 = true;
+    boolean canBeShiftJIS = true;
+    boolean sawDoubleByteStart = false;
+    int maybeSingleByteKatakanaCount = 0;
     boolean lastWasPossibleDoubleByteStart = false;
-    for (int i = 0; i < length; i++) {
+    for (int i = 0; i < length && (canBeISO88591 || canBeShiftJIS); i++) {
       int value = bytes[i] & 0xFF;
-      if (value >= 0x80 && value <= 0x9F && i < length - 1) {
+      if (value >= 0x7F && value <= 0x9F) {
         canBeISO88591 = false;
-        // ISO-8859-1 shouldn't use this, but before we decide it is Shift_JIS,
-        // just double check that it is followed by a byte that's valid in
-        // the Shift_JIS encoding
+      }
+      if (value >= 0xA1 && value <= 0xDF) {
+        // count the number of characters that might be a Shift_JIS single-byte Katakana character
+        maybeSingleByteKatakanaCount++;
+      }
+      if (((value >= 0x81 && value <= 0x9F) || (value >= 0xE0 && value <= 0xEF)) && i < length - 1) {
+        // These start double-byte characters in Shift_JIS. Let's see if it's followed by a valid
+        // second byte.
+        sawDoubleByteStart = true;
         if (lastWasPossibleDoubleByteStart) {
           // If we just checked this and the last byte for being a valid double-byte
           // char, don't check starting on this byte. If this and the last byte
@@ -274,30 +283,27 @@ final class DecodedBitStreamParser {
           lastWasPossibleDoubleByteStart = true;
           int nextValue = bytes[i + 1] & 0xFF;
           if (nextValue < 0x40 || nextValue > 0xFC) {
-            return UTF8;
+            canBeShiftJIS = false;
           }
           // There is some conflicting information out there about which bytes can follow which in
           // double-byte Shift_JIS characters. The rule above seems to be the one that matches practice.
-          // The stricter rule below, however, is given by other resources.
-          /*
-          if ((value & 0x1) == 0) {
-            // if even, next value should be in [0x9F,0xFC]
-            // if not, we'll guess UTF-8
-            if (nextValue < 0x9F || nextValue > 0xFC) {
-              return UTF8;
-            }
-          } else {
-            // if odd, next value should be in [0x40,0x9E]
-            // if not, we'll guess UTF-8
-            if (nextValue < 0x40 || nextValue > 0x9E) {
-              return UTF8;
-            }
-          }
-           */
         }
       }
     }
-    return canBeISO88591 ? ISO88591 : SHIFT_JIS;
+    // Distinguishing Shift_JIS and ISO-8859-1 can be a little tough. The crude heuristic is:
+    // - If we saw
+    //   - at least one byte that starts a double-byte value (bytes that are rare in ISO-8859-1), or
+    //   - over 5% of bytes that could be single-byte Katakana (also rare in ISO-8859-1),
+    // - and, saw no sequences that are invalid in Shift_JIS, then we conclude Shift_JIS
+    if ((sawDoubleByteStart || 20 * maybeSingleByteKatakanaCount > length) && canBeShiftJIS) {
+      return SHIFT_JIS;
+    }
+    // Otherwise, we default to ISO-8859-1 unless we know it can't be
+    if (canBeISO88591) {
+      return ISO88591;
+    }
+    // Otherwise, we take a wild guess with UTF-8
+    return UTF8;
   }
   
   private static int parseECIValue(BitSource bits) {

diff --git a/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java b/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java
index a5d946f..82bfa4e 100644 (file)
--- a/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java
+++ b/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java
@@ -29,10 +29,10 @@ public final class QRCodeBlackBox2TestCase extends AbstractBlackBoxTestCase {
 
   public QRCodeBlackBox2TestCase() {
     super(new File("test/data/blackbox/qrcode-2"), new MultiFormatReader(), BarcodeFormat.QR_CODE);
-    addTest(14, 14, 0.0f);
-    addTest(10, 10, 90.0f);
-    addTest(13, 13, 180.0f);
-    addTest(10, 10, 270.0f);
+    addTest(23, 23, 0.0f);
+    addTest(18, 18, 90.0f);
+    addTest(22, 22, 180.0f);
+    addTest(17, 17, 270.0f);
   }
 
 }
\ No newline at end of file

diff --git a/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java b/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java
index 8b41366..76be390 100644 (file)
--- a/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java
+++ b/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java
@@ -28,6 +28,17 @@ import junit.framework.TestCase;
 public final class DecodedBitStreamParserTestCase extends TestCase {
 
   public void testSimpleByteMode() throws ReaderException {
+    BitSourceBuilder builder = new BitSourceBuilder();
+    builder.write(0x04, 4); // Byte mode
+    builder.write(0x03, 8); // 3 bytes
+    builder.write(0xF1, 8);
+    builder.write(0xF2, 8);
+    builder.write(0xF3, 8);
+    String result = DecodedBitStreamParser.decode(builder.toByteArray(), Version.getVersionForNumber(1));
+    assertEquals("\u00f1\u00f2\u00f3", result);
+  }
+
+  public void testSimpleSJIS() throws ReaderException {
     BitSourceBuilder builder = new BitSourceBuilder();
     builder.write(0x04, 4); // Byte mode
     builder.write(0x03, 8); // 3 bytes
@@ -35,7 +46,7 @@ public final class DecodedBitStreamParserTestCase extends TestCase {
     builder.write(0xA2, 8);
     builder.write(0xA3, 8);
     String result = DecodedBitStreamParser.decode(builder.toByteArray(), Version.getVersionForNumber(1));
-    assertEquals("\u00a1\u00a2\u00a3", result); // this should be "¡¢£" if your editor character encoding matches mine!
+    assertEquals("\uff61\uff62\uff63", result);
   }
 
   public void testECI() throws ReaderException {
@@ -48,7 +59,7 @@ public final class DecodedBitStreamParserTestCase extends TestCase {
     builder.write(0xA2, 8);
     builder.write(0xA3, 8);
     String result = DecodedBitStreamParser.decode(builder.toByteArray(), Version.getVersionForNumber(1));
-    assertEquals("\u00ed\u00f3\u00fa", result); // should be like "íóú"
+    assertEquals("\u00ed\u00f3\u00fa", result);
   }
 
   // TODO definitely need more tests here