| /* | |
| * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. | |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
| * | |
| * This code is free software; you can redistribute it and/or modify it | |
| * under the terms of the GNU General Public License version 2 only, as | |
| * published by the Free Software Foundation. | |
| * | |
| * This code is distributed in the hope that it will be useful, but WITHOUT | |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| * version 2 for more details (a copy is included in the LICENSE file that | |
| * accompanied this code). | |
| * | |
| * You should have received a copy of the GNU General Public License version | |
| * 2 along with this work; if not, write to the Free Software Foundation, | |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| * | |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |
| * or visit www.oracle.com if you need additional information or have any | |
| * questions. | |
| */ | |
| /* | |
| * test | |
| * bug 4221795 | |
| * summary Confirm *.icu data using ICU4J Normalizer | |
| */ | |
| import java.io.BufferedReader; | |
| import java.io.FileInputStream; | |
| import java.io.InputStreamReader; | |
| import java.nio.charset.Charset; | |
| import java.nio.charset.CharsetDecoder; | |
| import java.util.BitSet; | |
| import java.util.StringTokenizer; | |
| import com.ibm.icu.text.Normalizer; | |
| import com.ibm.icu.impl.NormalizerImpl; | |
| /** | |
| * This is not a test program but a data validation utility. | |
| * Two datafiles for Normalizer, unorm.icu and uprops.icu under | |
| * sun/text/resouces, are generated using generators in ICU4C 3.2 on a | |
| * BIG-ENDIAN machine. Before using them with java.text.Normalizer and | |
| * sun.text.Normalizer, you may want to check these test datafile's validation. | |
| * You can test datafiles using Normalizer in ICU4J 3.2. Download ICU4J 3.2 and | |
| * run this test program with -cp <ICU4J 3.2>. | |
| */ | |
| public class DataValidationTest { | |
| // | |
| // Options to be used with com.ibm.icu.text.Normalizer | |
| // | |
| /* | |
| * Default Unicode 3.2.0 normalization. | |
| * | |
| * - With Corrigendum 4 fix | |
| * (Different from Mustang's Normalizer.) | |
| * - With Public Review Issue #29 fix | |
| * (Different from Mustang's Normalizer.) | |
| */ | |
| private static final int UNICODE_3_2_0 = Normalizer.UNICODE_3_2; | |
| /* | |
| * *Incomplete* Unicode 3.2.0 normalization for IDNA/StringPrep. | |
| * | |
| * - With Corrigendum 4 fix | |
| * - Without Public Review Issue #29 fix | |
| * | |
| * ICU4J's Normalizer itself doesn't support normalization for Unicode 3.2.0 | |
| * without Corrigendum 4 fix, which is necessary for IDNA/StringPrep. It is | |
| * done in StringPrep. Therefore, we don't test the normlaization in this | |
| * test program. We merely test normalization for Unicode 3.2.0 without | |
| * Public Review Issue #29 fix with this test program. | |
| */ | |
| private static final int UNICODE_3_2_0_BEFORE_PRI_29 = | |
| Normalizer.UNICODE_3_2 | | |
| NormalizerImpl.BEFORE_PRI_29; | |
| /* | |
| * Default normalization. | |
| * | |
| * - Unicode 4.0.1 | |
| * (Different from Mustang's Normalizer.) | |
| * - With Corrigendum 4 fix | |
| * - With Public Review Issue #29 fix | |
| * (Different from Mustang's Normalizer.) | |
| * | |
| * Because Public Review Issue #29 is fixed in Unicode 4.1.0. I think that | |
| * IUC4J 3.2 should not support it. But it actually supports PRI #29 fix | |
| * as default.... | |
| */ | |
| private static final int UNICODE_LATEST = 0x00; | |
| /* | |
| * Normalization without Public Review Issue #29 fix. | |
| * | |
| * - Unicode 4.0.1 | |
| * - Without Corrigendum 4 fix | |
| * - Without Public Review Issue #29 fix | |
| */ | |
| static final int UNICODE_LATEST_BEFORE_PRI_29 = | |
| NormalizerImpl.BEFORE_PRI_29; | |
| // | |
| // Conformance test datafiles | |
| // | |
| /* | |
| * Conformance test datafile for normalization for Unicode 3.2.0 with | |
| * Corrigendum 4 corrections. This is NOT an original Conformace test | |
| * data. Some inconvenient test cases are commented out. | |
| * About corrigendum 4, please refer | |
| * http://www.unicode.org/versions/corrigendum4.html | |
| * | |
| * ICU4J 3.2's Normalizer itself doesn't support normalization for Unicode | |
| * 3.2.0 without Corrigendum 4 corrections. StringPrep helps it. So, we | |
| * don't test the normalization with this test program. | |
| */ | |
| static final String DATA_3_2_0 = "NormalizationTest-3.2.0.Corrigendum4.txt"; | |
| /* | |
| * Conformance test datafile for the latest Unicode which is supported | |
| * by J2SE. | |
| */ | |
| static final String DATA_LATEST = "NormalizationTest-Latest.txt"; | |
| /* | |
| * Decorder | |
| */ | |
| static final CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder(); | |
| /* | |
| * List to pick up characters which are not listed in Part1 | |
| */ | |
| static BitSet charList = new BitSet(Character.MAX_CODE_POINT+1); | |
| /* | |
| * Shortcuts | |
| */ | |
| static final Normalizer.Mode NFC = com.ibm.icu.text.Normalizer.NFC; | |
| static final Normalizer.Mode NFD = com.ibm.icu.text.Normalizer.NFD; | |
| static final Normalizer.Mode NFKC = com.ibm.icu.text.Normalizer.NFKC; | |
| static final Normalizer.Mode NFKD = com.ibm.icu.text.Normalizer.NFKD; | |
| static final Normalizer.Mode[] modes = {NFC, NFD, NFKC, NFKD}; | |
| public static void main(String[] args) throws Exception { | |
| test(DATA_3_2_0, UNICODE_3_2_0); | |
| test(DATA_3_2_0, UNICODE_3_2_0_BEFORE_PRI_29); | |
| test(DATA_LATEST, UNICODE_LATEST); | |
| // This test started failing since ICU4J 3.6. | |
| // test(DATA_LATEST, UNICODE_LATEST_BEFORE_PRI_29); | |
| /* Unconformity test */ | |
| // test(DATA_3_2_0, UNICODE_LATEST); | |
| // test(DATA_LATEST, UNICODE_3_2); | |
| } | |
| private static void test(String filename, int unicodeVer) throws Exception { | |
| FileInputStream fis = new FileInputStream(filename); | |
| BufferedReader in = | |
| new BufferedReader(new InputStreamReader(fis, decoder)); | |
| System.out.println("\nStart testing with " + filename + | |
| " for options: " + | |
| (((unicodeVer & Normalizer.UNICODE_3_2) != 0) ? | |
| "Unicode 3.2.0" : "the latest Unicode") + ", " + | |
| (((unicodeVer & NormalizerImpl.BEFORE_PRI_29) != 0) ? | |
| "with" : "without") + " PRI #29 fix"); | |
| int lineNo = 0; | |
| String text; | |
| String[] columns = new String[6]; | |
| boolean part1test = false; | |
| while ((text = in.readLine()) != null) { | |
| lineNo ++; | |
| char c = text.charAt(0); | |
| if (c == '#') { | |
| continue; | |
| } else if (c == '@') { | |
| if (text.startsWith("@Part")) { | |
| System.out.println("# Testing data in " + text); | |
| if (text.startsWith("@Part1 ")) { | |
| part1test = true; | |
| } else { | |
| part1test = false; | |
| } | |
| continue; | |
| } | |
| } | |
| prepareColumns(columns, text, filename, lineNo, part1test); | |
| testNFC(columns, unicodeVer, filename, lineNo); | |
| testNFD(columns, unicodeVer, filename, lineNo); | |
| testNFKC(columns, unicodeVer, filename, lineNo); | |
| testNFKD(columns, unicodeVer, filename, lineNo); | |
| } | |
| in.close(); | |
| fis.close(); | |
| if (unicodeVer == UNICODE_LATEST) { | |
| System.out.println("# Testing characters which are not listed in Part1"); | |
| testRemainingChars(filename, unicodeVer); | |
| } | |
| } | |
| /* | |
| * Test for NFC | |
| * | |
| * c2 == NFC(c1) == NFC(c2) == NFC(c3) | |
| * c4 == NFC(c4) == NFC(c5) | |
| */ | |
| private static void testNFC(String[] c, int unicodeVer, | |
| String file, int line) throws Exception { | |
| test(2, c, 1, 3, NFC, unicodeVer, file, line); | |
| test(4, c, 4, 5, NFC, unicodeVer, file, line); | |
| } | |
| /* | |
| * Test for NFD | |
| * | |
| * c3 == NFD(c1) == NFD(c2) == NFD(c3) | |
| * c5 == NFD(c4) == NFD(c5) | |
| */ | |
| private static void testNFD(String[] c, int unicodeVer, | |
| String file, int line) throws Exception { | |
| test(3, c, 1, 3, NFD, unicodeVer, file, line); | |
| test(5, c, 4, 5, NFD, unicodeVer, file, line); | |
| } | |
| /* | |
| * Test for NFKC | |
| * | |
| * c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5) | |
| */ | |
| private static void testNFKC(String[] c, int unicodeVer, | |
| String file, int line) throws Exception { | |
| test(4, c, 1, 5, NFKC, unicodeVer, file, line); | |
| } | |
| /* | |
| * Test for NFKD | |
| * | |
| * c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5) | |
| */ | |
| private static void testNFKD(String[] c, int unicodeVer, | |
| String file, int line) throws Exception { | |
| test(5, c, 1, 5, NFKD, unicodeVer, file, line); | |
| } | |
| /* | |
| * Test for characters which aren't listed in Part1 | |
| * | |
| * X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X) | |
| */ | |
| private static void testRemainingChars(String file, | |
| int unicodeVer) throws Exception { | |
| for (int i = Character.MIN_CODE_POINT; | |
| i <= Character.MAX_CODE_POINT; | |
| i++) { | |
| if (!charList.get(i)) { | |
| String from = String.valueOf(Character.toChars(i)); | |
| String to; | |
| for (int j = 0; j < modes.length; j++) { | |
| Normalizer.Mode mode = modes[j]; | |
| to = Normalizer.normalize(from, mode, unicodeVer); | |
| if (!from.equals(to)) { | |
| error(mode, from, from, to, file, -1); | |
| // } else { | |
| // okay(mode, from, from, to, file, -1); | |
| } | |
| if (!Normalizer.isNormalized(from, mode, unicodeVer)) { | |
| error(mode, from, file, -1); | |
| // } else { | |
| // okay(mode, from, file, -1); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| /* | |
| * Test normalize() and isNormalized() | |
| */ | |
| private static void test(int col, String[] c, | |
| int FROM, int TO, | |
| Normalizer.Mode mode, int unicodeVer, | |
| String file, int line) throws Exception { | |
| for (int i = FROM; i <= TO; i++) { | |
| String got = Normalizer.normalize(c[i], mode, unicodeVer); | |
| if (!c[col].equals(got)) { | |
| error(mode, c[i], c[col], got, file, line); | |
| // } else { | |
| // okay(mode, c[i], c[col], got, file, line); | |
| } | |
| /* | |
| * If the original String equals its normalized String, it means | |
| * that the original String is normalizerd. Thus, isNormalized() | |
| * should return true. And, vice versa! | |
| */ | |
| if (c[col].equals(c[i])) { | |
| if (!Normalizer.isNormalized(c[i], mode, unicodeVer)) { | |
| error(mode, c[i], file, line); | |
| // } else { | |
| // okay(mode, c[i], file, line); | |
| } | |
| } else { | |
| if (Normalizer.isNormalized(c[i], mode, unicodeVer)) { | |
| error(mode, c[i], file, line); | |
| // } else { | |
| // okay(mode, c[i], file, line); | |
| } | |
| } | |
| } | |
| } | |
| /* | |
| * Generate an array of String from a line of conformance datafile. | |
| */ | |
| private static void prepareColumns(String[] col, String text, | |
| String file, int line, | |
| boolean part1test) throws Exception { | |
| int index = text.indexOf('#'); | |
| if (index != -1) { | |
| text = text.substring(0, index); | |
| } | |
| StringTokenizer st = new StringTokenizer(text, ";"); | |
| int tokenCount = st.countTokens(); | |
| if (tokenCount < 5) { | |
| throw new RuntimeException("# of tokens in datafile should be 6, but got: " + tokenCount + " at line " + line + " in " + file); | |
| } | |
| StringBuffer sb = new StringBuffer(); | |
| for (int i = 1; i <= 5; i++) { | |
| StringTokenizer tst = new StringTokenizer(st.nextToken(), " "); | |
| while (tst.hasMoreTokens()) { | |
| int code = Integer.parseInt(tst.nextToken(), 16); | |
| sb.append(Character.toChars(code)); | |
| } | |
| col[i] = sb.toString(); | |
| sb.setLength(0); | |
| } | |
| if (part1test) { | |
| charList.set(col[1].codePointAt(0)); | |
| } | |
| } | |
| /* | |
| * Show an error message when normalize() didn't return the expected value. | |
| * (An exception is sometimes convenient. Therefore, it is commented out | |
| * for the moment.) | |
| */ | |
| private static void error(Normalizer.Mode mode, | |
| String from, String to, String got, | |
| String file, int line) throws Exception { | |
| System.err.println("\t" + toString(mode) + ": normalize(" + | |
| toHexString(from) + ") doesn't equal <" + toHexString(to) + | |
| "> at line " + line + " in " + file + ". Got <" + | |
| toHexString(got) + ">."); | |
| // throw new RuntimeException("Normalization(" + toString(mode) + ") failed"); | |
| } | |
| /* | |
| * Show an error message when isNormalize() didn't return the expected value. | |
| * (An exception is sometimes convenient. Therefore, it is commented out | |
| * for the moment.) | |
| */ | |
| private static void error(Normalizer.Mode mode, String orig, | |
| String file, int line) throws Exception { | |
| System.err.println("\t" + toString(mode) + ": isNormalized(" + | |
| toHexString(orig) + ") returned the wrong value at line " + line + | |
| " in " + file + "."); | |
| // throw new RuntimeException("Normalization(" + toString(mode) +") failed"); | |
| } | |
| /* | |
| * (For debugging) | |
| * Shows a message when normalize() returned the expected value. | |
| */ | |
| private static void okay(Normalizer.Mode mode, | |
| String from, String to, String got, | |
| String file, int line) { | |
| System.out.println("\t" + toString(mode) + ": normalize(" + | |
| toHexString(from) + ") equals <" + toHexString(to) + | |
| "> at line " + line + " in " + file + ". Got <" + | |
| toHexString(got) + ">."); | |
| } | |
| /* | |
| * (For debugging) | |
| * Shows a message when isNormalized() returned the expected value. | |
| */ | |
| private static void okay(Normalizer.Mode mode, String orig, | |
| String file, int line) { | |
| System.out.println("\t" + toString(mode) + ": isNormalized(" + | |
| toHexString(orig) + ") returned the correct value at line " + | |
| line + " in " + file + "."); | |
| } | |
| /* | |
| * Returns a spece-delimited hex String | |
| */ | |
| private static String toHexString(String s) { | |
| StringBuffer sb = new StringBuffer(" "); | |
| for (int i = 0; i < s.length(); i++) { | |
| sb.append(Integer.toHexString(s.charAt(i))); | |
| sb.append(' '); | |
| } | |
| return sb.toString(); | |
| } | |
| /* | |
| * Returns the name of Normalizer.Mode | |
| */ | |
| private static String toString(Normalizer.Mode mode) { | |
| if (mode == Normalizer.NFC) { | |
| return "NFC"; | |
| } else if (mode == Normalizer.NFD) { | |
| return "NFD"; | |
| } else if (mode == Normalizer.NFKC) { | |
| return "NFKC"; | |
| } else if (mode == Normalizer.NFKD) { | |
| return "NFKD"; | |
| } | |
| return "unknown"; | |
| } | |
| } |