| /* | |
| * Copyright (c) 1996, 2023, Oracle and/or its affiliates. All rights reserved. | |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
| * | |
| * This code is free software; you can redistribute it and/or modify it | |
| * under the terms of the GNU General Public License version 2 only, as | |
| * published by the Free Software Foundation. | |
| * | |
| * This code is distributed in the hope that it will be useful, but WITHOUT | |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| * version 2 for more details (a copy is included in the LICENSE file that | |
| * accompanied this code). | |
| * | |
| * You should have received a copy of the GNU General Public License version | |
| * 2 along with this work; if not, write to the Free Software Foundation, | |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| * | |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |
| * or visit www.oracle.com if you need additional information or have any | |
| * questions. | |
| */ | |
| /* | |
| * @test | |
| * @bug 4035266 4052418 4068133 4068137 4068139 4086052 4095322 4097779 | |
| * 4097920 4098467 4111338 4113835 4117554 4143071 4146175 4152117 | |
| * 4152416 4153072 4158381 4214367 4217703 4638433 8264765 8291660 | |
| * 8294008 | |
| * @run junit/timeout=2000 BreakIteratorTest | |
| * @summary test BreakIterator | |
| */ | |
| /* | |
| * This file is available under and governed by the GNU General Public | |
| * License version 2 only, as published by the Free Software Foundation. | |
| * However, the following notice accompanied the original version of this | |
| * file and, per its terms, should not be removed: | |
| * | |
| * (C) Copyright Taligent, Inc. 1996, 1997 - All Rights Reserved | |
| * (C) Copyright IBM Corp. 1996 - 1998 - All Rights Reserved | |
| * | |
| * Portions copyright (c) 2007 Sun Microsystems, Inc. | |
| * All Rights Reserved. | |
| * | |
| * The original version of this source code and documentation | |
| * is copyrighted and owned by Taligent, Inc., a wholly-owned | |
| * subsidiary of IBM. These materials are provided under terms | |
| * of a License Agreement between Taligent and Sun. This technology | |
| * is protected by multiple US and International patents. | |
| * | |
| * This notice and attribution to Taligent may not be removed. | |
| * Taligent is a registered trademark of Taligent, Inc. | |
| * | |
| * Permission to use, copy, modify, and distribute this software | |
| * and its documentation for NON-COMMERCIAL purposes and without | |
| * fee is hereby granted provided that this copyright notice | |
| * appears in all copies. Please refer to the file "copyright.html" | |
| * for further important copyright and licensing information. | |
| * | |
| * SUN MAKES NO REPRESENTATIONS OR WARRANTIES ABOUT THE SUITABILITY OF | |
| * THE SOFTWARE, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED | |
| * TO THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A | |
| * PARTICULAR PURPOSE, OR NON-INFRINGEMENT. SUN SHALL NOT BE LIABLE FOR | |
| * ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF USING, MODIFYING OR | |
| * DISTRIBUTING THIS SOFTWARE OR ITS DERIVATIVES. | |
| * | |
| */ | |
| import java.nio.file.Files; | |
| import java.nio.file.Paths; | |
| import java.text.BreakIterator; | |
| import java.text.StringCharacterIterator; | |
| import java.util.Arrays; | |
| import java.util.Locale; | |
| import java.util.Vector; | |
| import java.util.function.Predicate; | |
| import java.util.regex.Pattern; | |
| import org.junit.jupiter.api.Test; | |
| import static org.junit.jupiter.api.Assertions.fail; | |
| public class BreakIteratorTest { | |
| private final BreakIterator characterBreak = BreakIterator.getCharacterInstance(); | |
| private final BreakIterator wordBreak = BreakIterator.getWordInstance(); | |
| private final BreakIterator lineBreak = BreakIterator.getLineInstance(); | |
| private final BreakIterator sentenceBreak = BreakIterator.getSentenceInstance(); | |
| //========================================================================= | |
| // general test subroutines | |
| //========================================================================= | |
| private void generalIteratorTest(BreakIterator bi, Vector expectedResult) { | |
| StringBuffer buffer = new StringBuffer(); | |
| String text; | |
| for (int i = 0; i < expectedResult.size(); i++) { | |
| text = (String)expectedResult.elementAt(i); | |
| buffer.append(text); | |
| } | |
| text = buffer.toString(); | |
| bi.setText(text); | |
| Vector nextResults = testFirstAndNext(bi, text); | |
| Vector previousResults = testLastAndPrevious(bi, text); | |
| System.out.println("comparing forward and backward..."); | |
| compareFragmentLists("forward iteration", "backward iteration", nextResults, | |
| previousResults); | |
| System.out.println("comparing expected and actual..."); | |
| compareFragmentLists("expected result", "actual result", expectedResult, | |
| nextResults); | |
| int[] boundaries = new int[expectedResult.size() + 3]; | |
| boundaries[0] = BreakIterator.DONE; | |
| boundaries[1] = 0; | |
| for (int i = 0; i < expectedResult.size(); i++) | |
| boundaries[i + 2] = boundaries[i + 1] + ((String)expectedResult.elementAt(i)). | |
| length(); | |
| boundaries[boundaries.length - 1] = BreakIterator.DONE; | |
| testFollowing(bi, text, boundaries); | |
| testPreceding(bi, text, boundaries); | |
| testIsBoundary(bi, text, boundaries); | |
| doMultipleSelectionTest(bi, text); | |
| } | |
| private Vector testFirstAndNext(BreakIterator bi, String text) { | |
| int p = bi.first(); | |
| int lastP = p; | |
| Vector<String> result = new Vector<String>(); | |
| if (p != 0) | |
| fail("first() returned " + p + " instead of 0"); | |
| while (p != BreakIterator.DONE) { | |
| p = bi.next(); | |
| if (p != BreakIterator.DONE) { | |
| if (p <= lastP) | |
| fail("next() failed to move forward: next() on position " | |
| + lastP + " yielded " + p); | |
| result.addElement(text.substring(lastP, p)); | |
| } | |
| else { | |
| if (lastP != text.length()) | |
| fail("next() returned DONE prematurely: offset was " | |
| + lastP + " instead of " + text.length()); | |
| } | |
| lastP = p; | |
| } | |
| return result; | |
| } | |
| private Vector testLastAndPrevious(BreakIterator bi, String text) { | |
| int p = bi.last(); | |
| int lastP = p; | |
| Vector<String> result = new Vector<String>(); | |
| if (p != text.length()) | |
| fail("last() returned " + p + " instead of " + text.length()); | |
| while (p != BreakIterator.DONE) { | |
| p = bi.previous(); | |
| if (p != BreakIterator.DONE) { | |
| if (p >= lastP) | |
| fail("previous() failed to move backward: previous() on position " | |
| + lastP + " yielded " + p); | |
| result.insertElementAt(text.substring(p, lastP), 0); | |
| } | |
| else { | |
| if (lastP != 0) | |
| fail("previous() returned DONE prematurely: offset was " | |
| + lastP + " instead of 0"); | |
| } | |
| lastP = p; | |
| } | |
| return result; | |
| } | |
| private void compareFragmentLists(String f1Name, String f2Name, Vector f1, Vector f2) { | |
| int p1 = 0; | |
| int p2 = 0; | |
| String s1; | |
| String s2; | |
| int t1 = 0; | |
| int t2 = 0; | |
| while (p1 < f1.size() && p2 < f2.size()) { | |
| s1 = (String)f1.elementAt(p1); | |
| s2 = (String)f2.elementAt(p2); | |
| t1 += s1.length(); | |
| t2 += s2.length(); | |
| if (s1.equals(s2)) { | |
| debugLogln(" >" + s1 + "<"); | |
| ++p1; | |
| ++p2; | |
| } | |
| else { | |
| int tempT1 = t1; | |
| int tempT2 = t2; | |
| int tempP1 = p1; | |
| int tempP2 = p2; | |
| while (tempT1 != tempT2 && tempP1 < f1.size() && tempP2 < f2.size()) { | |
| while (tempT1 < tempT2 && tempP1 < f1.size()) { | |
| tempT1 += ((String)f1.elementAt(tempP1)).length(); | |
| ++tempP1; | |
| } | |
| while (tempT2 < tempT1 && tempP2 < f2.size()) { | |
| tempT2 += ((String)f2.elementAt(tempP2)).length(); | |
| ++tempP2; | |
| } | |
| } | |
| System.out.println("*** " + f1Name + " has:"); | |
| while (p1 <= tempP1 && p1 < f1.size()) { | |
| s1 = (String)f1.elementAt(p1); | |
| t1 += s1.length(); | |
| debugLogln(" *** >" + s1 + "<"); | |
| ++p1; | |
| } | |
| System.out.println("***** " + f2Name + " has:"); | |
| while (p2 <= tempP2 && p2 < f2.size()) { | |
| s2 = (String)f2.elementAt(p2); | |
| t2 += s2.length(); | |
| debugLogln(" ***** >" + s2 + "<"); | |
| ++p2; | |
| } | |
| fail("Discrepancy between " + f1Name + " and " + f2Name | |
| + "\n---\n" + f1 +"\n---\n" + f2); | |
| } | |
| } | |
| } | |
| private void testFollowing(BreakIterator bi, String text, int[] boundaries) { | |
| System.out.println("testFollowing():"); | |
| int p = 2; | |
| int i = 0; | |
| try { | |
| for (i = 0; i <= text.length(); i++) { // change to <= when new BI code goes in | |
| if (i == boundaries[p]) | |
| ++p; | |
| int b = bi.following(i); | |
| System.out.println("bi.following(" + i + ") -> " + b); | |
| if (b != boundaries[p]) | |
| fail("Wrong result from following() for " + i + ": expected " + boundaries[p] | |
| + ", got " + b); | |
| } | |
| } catch (IllegalArgumentException illargExp) { | |
| fail("IllegalArgumentException caught from following() for offset: " + i); | |
| } | |
| } | |
| private void testPreceding(BreakIterator bi, String text, int[] boundaries) { | |
| System.out.println("testPreceding():"); | |
| int p = 0; | |
| int i = 0; | |
| try { | |
| for (i = 0; i <= text.length(); i++) { // change to <= when new BI code goes in | |
| int b = bi.preceding(i); | |
| System.out.println("bi.preceding(" + i + ") -> " + b); | |
| if (b != boundaries[p]) | |
| fail("Wrong result from preceding() for " + i + ": expected " + boundaries[p] | |
| + ", got " + b); | |
| if (i == boundaries[p + 1]) | |
| ++p; | |
| } | |
| } catch (IllegalArgumentException illargExp) { | |
| fail("IllegalArgumentException caught from preceding() for offset: " + i); | |
| } | |
| } | |
| private void testIsBoundary(BreakIterator bi, String text, int[] boundaries) { | |
| System.out.println("testIsBoundary():"); | |
| int p = 1; | |
| boolean isB; | |
| for (int i = 0; i <= text.length(); i++) { // change to <= when new BI code goes in | |
| isB = bi.isBoundary(i); | |
| System.out.println("bi.isBoundary(" + i + ") -> " + isB); | |
| if (i == boundaries[p]) { | |
| if (!isB) | |
| fail("Wrong result from isBoundary() for " + i + ": expected true, got false"); | |
| ++p; | |
| } | |
| else { | |
| if (isB) | |
| fail("Wrong result from isBoundary() for " + i + ": expected false, got true"); | |
| } | |
| } | |
| } | |
| private void doMultipleSelectionTest(BreakIterator iterator, String testText) | |
| { | |
| System.out.println("Multiple selection test..."); | |
| BreakIterator testIterator = (BreakIterator)iterator.clone(); | |
| int offset = iterator.first(); | |
| int testOffset; | |
| int count = 0; | |
| do { | |
| testOffset = testIterator.first(); | |
| testOffset = testIterator.next(count); | |
| System.out.println("next(" + count + ") -> " + testOffset); | |
| if (offset != testOffset) | |
| fail("next(n) and next() not returning consistent results: for step " + count + ", next(n) returned " + testOffset + " and next() had " + offset); | |
| if (offset != BreakIterator.DONE) { | |
| count++; | |
| offset = iterator.next(); | |
| } | |
| } while (offset != BreakIterator.DONE); | |
| // now do it backwards... | |
| offset = iterator.last(); | |
| count = 0; | |
| do { | |
| testOffset = testIterator.last(); | |
| testOffset = testIterator.next(count); | |
| System.out.println("next(" + count + ") -> " + testOffset); | |
| if (offset != testOffset) | |
| fail("next(n) and next() not returning consistent results: for step " + count + ", next(n) returned " + testOffset + " and next() had " + offset); | |
| if (offset != BreakIterator.DONE) { | |
| count--; | |
| offset = iterator.previous(); | |
| } | |
| } while (offset != BreakIterator.DONE); | |
| } | |
| private void doBreakInvariantTest(BreakIterator tb, String testChars) | |
| { | |
| StringBuffer work = new StringBuffer("aaa"); | |
| int errorCount = 0; | |
| // a break should always occur after CR (unless followed by LF), LF, PS, and LS | |
| String breaks = /*"\r\n\u2029\u2028"*/"\n\u2029\u2028"; | |
| // change this back when new BI code is added | |
| for (int i = 0; i < breaks.length(); i++) { | |
| work.setCharAt(1, breaks.charAt(i)); | |
| for (int j = 0; j < testChars.length(); j++) { | |
| work.setCharAt(0, testChars.charAt(j)); | |
| for (int k = 0; k < testChars.length(); k++) { | |
| char c = testChars.charAt(k); | |
| // if a cr is followed by lf, don't do the check (they stay together) | |
| if (work.charAt(1) == '\r' && (c == '\n')) | |
| continue; | |
| // CONTROL (Cc) and FORMAT (Cf) Characters are to be ignored | |
| // for breaking purposes as per UTR14 | |
| int type1 = Character.getType(work.charAt(1)); | |
| int type2 = Character.getType(c); | |
| if (type1 == Character.CONTROL || type1 == Character.FORMAT || | |
| type2 == Character.CONTROL || type2 == Character.FORMAT) { | |
| continue; | |
| } | |
| work.setCharAt(2, c); | |
| tb.setText(work.toString()); | |
| boolean seen2 = false; | |
| for (int l = tb.first(); l != BreakIterator.DONE; l = tb.next()) { | |
| if (l == 2) | |
| seen2 = true; | |
| } | |
| if (!seen2) { | |
| fail("No break between U+" + Integer.toHexString((int)(work.charAt(1))) | |
| + " and U+" + Integer.toHexString((int)(work.charAt(2)))); | |
| errorCount++; | |
| if (errorCount >= 75) | |
| return; | |
| } | |
| } | |
| } | |
| } | |
| } | |
| private void doOtherInvariantTest(BreakIterator tb, String testChars) | |
| { | |
| StringBuffer work = new StringBuffer("a\r\na"); | |
| int errorCount = 0; | |
| // a break should never occur between CR and LF | |
| for (int i = 0; i < testChars.length(); i++) { | |
| work.setCharAt(0, testChars.charAt(i)); | |
| for (int j = 0; j < testChars.length(); j++) { | |
| work.setCharAt(3, testChars.charAt(j)); | |
| tb.setText(work.toString()); | |
| for (int k = tb.first(); k != BreakIterator.DONE; k = tb.next()) | |
| if (k == 2) { | |
| fail("Break between CR and LF in string U+" + Integer.toHexString( | |
| (int)(work.charAt(0))) + ", U+d U+a U+" + Integer.toHexString( | |
| (int)(work.charAt(3)))); | |
| errorCount++; | |
| if (errorCount >= 75) | |
| return; | |
| } | |
| } | |
| } | |
| // a break should never occur before a non-spacing mark, unless it's preceded | |
| // by a line terminator | |
| work.setLength(0); | |
| work.append("aaaa"); | |
| for (int i = 0; i < testChars.length(); i++) { | |
| char c = testChars.charAt(i); | |
| if (c == '\n' || c == '\r' || c == '\u2029' || c == '\u2028' || c == '\u0003') | |
| continue; | |
| work.setCharAt(1, c); | |
| for (int j = 0; j < testChars.length(); j++) { | |
| c = testChars.charAt(j); | |
| if (Character.getType(c) != Character.NON_SPACING_MARK && Character.getType(c) | |
| != Character.ENCLOSING_MARK) | |
| continue; | |
| work.setCharAt(2, c); | |
| // CONTROL (Cc) and FORMAT (Cf) Characters are to be ignored | |
| // for breaking purposes as per UTR14 | |
| int type1 = Character.getType(work.charAt(1)); | |
| int type2 = Character.getType(work.charAt(2)); | |
| if (type1 == Character.CONTROL || type1 == Character.FORMAT || | |
| type2 == Character.CONTROL || type2 == Character.FORMAT) { | |
| continue; | |
| } | |
| tb.setText(work.toString()); | |
| for (int k = tb.first(); k != BreakIterator.DONE; k = tb.next()) | |
| if (k == 2) { | |
| fail("Break between U+" + Integer.toHexString((int)(work.charAt(1))) | |
| + " and U+" + Integer.toHexString((int)(work.charAt(2)))); | |
| errorCount++; | |
| if (errorCount >= 75) | |
| return; | |
| } | |
| } | |
| } | |
| } | |
| public void debugLogln(String s) { | |
| final String zeros = "0000"; | |
| String temp; | |
| StringBuffer out = new StringBuffer(); | |
| for (int i = 0; i < s.length(); i++) { | |
| char c = s.charAt(i); | |
| if (c >= ' ' && c < '\u007f') | |
| out.append(c); | |
| else { | |
| out.append("\\u"); | |
| temp = Integer.toHexString((int)c); | |
| out.append(zeros.substring(0, 4 - temp.length())); | |
| out.append(temp); | |
| } | |
| } | |
| System.out.println(out.toString()); | |
| } | |
| //========================================================================= | |
| // tests | |
| //========================================================================= | |
| @Test | |
| public void TestWordBreak() { | |
| Vector<String> wordSelectionData = new Vector<String>(); | |
| wordSelectionData.addElement("12,34"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("\u00A2"); //cent sign | |
| wordSelectionData.addElement("\u00A3"); //pound sign | |
| wordSelectionData.addElement("\u00A4"); //currency sign | |
| wordSelectionData.addElement("\u00A5"); //yen sign | |
| wordSelectionData.addElement("alpha-beta-gamma"); | |
| wordSelectionData.addElement("."); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("Badges"); | |
| wordSelectionData.addElement("?"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("BADGES"); | |
| wordSelectionData.addElement("!"); | |
| wordSelectionData.addElement("?"); | |
| wordSelectionData.addElement("!"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("We"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("don't"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("need"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("no"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("STINKING"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("BADGES"); | |
| wordSelectionData.addElement("!"); | |
| wordSelectionData.addElement("!"); | |
| wordSelectionData.addElement("!"); | |
| wordSelectionData.addElement("012.566,5"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("123.3434,900"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("1000,233,456.000"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("1,23.322%"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("123.1222"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("\u0024123,000.20"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("179.01\u0025"); | |
| wordSelectionData.addElement("Hello"); | |
| wordSelectionData.addElement(","); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("how"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("are"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("you"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("X"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("Now"); | |
| wordSelectionData.addElement("\r"); | |
| wordSelectionData.addElement("is"); | |
| wordSelectionData.addElement("\n"); | |
| wordSelectionData.addElement("the"); | |
| wordSelectionData.addElement("\r\n"); | |
| wordSelectionData.addElement("time"); | |
| wordSelectionData.addElement("\n"); | |
| wordSelectionData.addElement("\r"); | |
| wordSelectionData.addElement("for"); | |
| wordSelectionData.addElement("\r"); | |
| wordSelectionData.addElement("\r"); | |
| wordSelectionData.addElement("all"); | |
| wordSelectionData.addElement(" "); | |
| generalIteratorTest(wordBreak, wordSelectionData); | |
| } | |
| @Test | |
| public void TestBug4097779() { | |
| Vector<String> wordSelectionData = new Vector<String>(); | |
| wordSelectionData.addElement("aa\u0300a"); | |
| wordSelectionData.addElement(" "); | |
| generalIteratorTest(wordBreak, wordSelectionData); | |
| } | |
| @Test | |
| public void TestBug4098467Words() { | |
| Vector<String> wordSelectionData = new Vector<String>(); | |
| // What follows is a string of Korean characters (I found it in the Yellow Pages | |
| // ad for the Korean Presbyterian Church of San Francisco, and I hope I transcribed | |
| // it correctly), first as precomposed syllables, and then as conjoining jamo. | |
| // Both sequences should be semantically identical and break the same way. | |
| // precomposed syllables... | |
| wordSelectionData.addElement("\uc0c1\ud56d"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("\ud55c\uc778"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("\uc5f0\ud569"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("\uc7a5\ub85c\uad50\ud68c"); | |
| wordSelectionData.addElement(" "); | |
| // conjoining jamo... | |
| wordSelectionData.addElement("\u1109\u1161\u11bc\u1112\u1161\u11bc"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("\u1112\u1161\u11ab\u110b\u1175\u11ab"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("\u110b\u1167\u11ab\u1112\u1161\u11b8"); | |
| wordSelectionData.addElement(" "); | |
| wordSelectionData.addElement("\u110c\u1161\u11bc\u1105\u1169\u1100\u116d\u1112\u116c"); | |
| wordSelectionData.addElement(" "); | |
| generalIteratorTest(wordBreak, wordSelectionData); | |
| } | |
| @Test | |
| public void TestBug4117554Words() { | |
| Vector<String> wordSelectionData = new Vector<String>(); | |
| // this is a test for bug #4117554: the ideographic iteration mark (U+3005) should | |
| // count as a Kanji character for the purposes of word breaking | |
| wordSelectionData.addElement("abc"); | |
| wordSelectionData.addElement("\u4e01\u4e02\u3005\u4e03\u4e03"); | |
| wordSelectionData.addElement("abc"); | |
| generalIteratorTest(wordBreak, wordSelectionData); | |
| } | |
| @Test | |
| public void TestSentenceBreak() { | |
| Vector<String> sentenceSelectionData = new Vector<String>(); | |
| sentenceSelectionData.addElement("This is a simple sample sentence. "); | |
| sentenceSelectionData.addElement("(This is it.) "); | |
| sentenceSelectionData.addElement("This is a simple sample sentence. "); | |
| sentenceSelectionData.addElement("\"This isn\'t it.\" "); | |
| sentenceSelectionData.addElement("Hi! "); | |
| sentenceSelectionData.addElement("This is a simple sample sentence. "); | |
| sentenceSelectionData.addElement("It does not have to make any sense as you can see. "); | |
| sentenceSelectionData.addElement("Nel mezzo del cammin di nostra vita, mi ritrovai in una selva oscura. "); | |
| sentenceSelectionData.addElement("Che la dritta via aveo smarrita. "); | |
| sentenceSelectionData.addElement("He said, that I said, that you said!! "); | |
| sentenceSelectionData.addElement("Don't rock the boat.\u2029"); | |
| sentenceSelectionData.addElement("Because I am the daddy, that is why. "); | |
| sentenceSelectionData.addElement("Not on my time (el timo.)! "); | |
| sentenceSelectionData.addElement("So what!!\u2029"); | |
| sentenceSelectionData.addElement("\"But now,\" he said, \"I know!\" "); | |
| sentenceSelectionData.addElement("Harris thumbed down several, including \"Away We Go\" (which became the huge success Oklahoma!). "); | |
| sentenceSelectionData.addElement("One species, B. anthracis, is highly virulent.\n"); | |
| sentenceSelectionData.addElement("Wolf said about Sounder:\"Beautifully thought-out and directed.\" "); | |
| sentenceSelectionData.addElement("Have you ever said, \"This is where \tI shall live\"? "); | |
| sentenceSelectionData.addElement("He answered, \"You may not!\" "); | |
| sentenceSelectionData.addElement("Another popular saying is: \"How do you do?\". "); | |
| sentenceSelectionData.addElement("Yet another popular saying is: \'I\'m fine thanks.\' "); | |
| sentenceSelectionData.addElement("What is the proper use of the abbreviation pp.? "); | |
| sentenceSelectionData.addElement("Yes, I am definatelly 12\" tall!!"); | |
| generalIteratorTest(sentenceBreak, sentenceSelectionData); | |
| } | |
| @Test | |
| public void TestBug4113835() { | |
| Vector<String> sentenceSelectionData = new Vector<String>(); | |
| // test for bug #4113835: \n and \r count as spaces, not as paragraph breaks | |
| sentenceSelectionData.addElement("Now\ris\nthe\r\ntime\n\rfor\r\rall\u2029"); | |
| generalIteratorTest(sentenceBreak, sentenceSelectionData); | |
| } | |
| @Test | |
| public void TestBug4111338() { | |
| Vector<String> sentenceSelectionData = new Vector<String>(); | |
| // test for bug #4111338: Don't break sentences at the boundary between CJK | |
| // and other letters | |
| sentenceSelectionData.addElement("\u5487\u67ff\ue591\u5017\u61b3\u60a1\u9510\u8165:\"JAVA\u821c" | |
| + "\u8165\u7fc8\u51ce\u306d,\u2494\u56d8\u4ec0\u60b1\u8560\u51ba" | |
| + "\u611d\u57b6\u2510\u5d46\".\u2029"); | |
| sentenceSelectionData.addElement("\u5487\u67ff\ue591\u5017\u61b3\u60a1\u9510\u8165\u9de8" | |
| + "\u97e4JAVA\u821c\u8165\u7fc8\u51ce\u306d\ue30b\u2494\u56d8\u4ec0" | |
| + "\u60b1\u8560\u51ba\u611d\u57b6\u2510\u5d46\u97e5\u7751\u2029"); | |
| sentenceSelectionData.addElement("\u5487\u67ff\ue591\u5017\u61b3\u60a1\u9510\u8165\u9de8\u97e4" | |
| + "\u6470\u8790JAVA\u821c\u8165\u7fc8\u51ce\u306d\ue30b\u2494\u56d8" | |
| + "\u4ec0\u60b1\u8560\u51ba\u611d\u57b6\u2510\u5d46\u97e5\u7751\u2029"); | |
| sentenceSelectionData.addElement("He said, \"I can go there.\"\u2029"); | |
| generalIteratorTest(sentenceBreak, sentenceSelectionData); | |
| } | |
| @Test | |
| public void TestBug4117554Sentences() { | |
| Vector<String> sentenceSelectionData = new Vector<String>(); | |
| // Treat fullwidth variants of .!? the same as their | |
| // normal counterparts | |
| sentenceSelectionData.addElement("I know I'm right\uff0e "); | |
| sentenceSelectionData.addElement("Right\uff1f "); | |
| sentenceSelectionData.addElement("Right\uff01 "); | |
| // Don't break sentences at boundary between CJK and digits | |
| sentenceSelectionData.addElement("\u5487\u67ff\ue591\u5017\u61b3\u60a1\u9510\u8165\u9de8" | |
| + "\u97e48888\u821c\u8165\u7fc8\u51ce\u306d\ue30b\u2494\u56d8\u4ec0" | |
| + "\u60b1\u8560\u51ba\u611d\u57b6\u2510\u5d46\u97e5\u7751\u2029"); | |
| // Break sentence between a sentence terminator and | |
| // opening punctuation | |
| sentenceSelectionData.addElement("no?"); | |
| sentenceSelectionData.addElement("(yes)"); | |
| generalIteratorTest(sentenceBreak, sentenceSelectionData); | |
| } | |
| @Test | |
| public void TestBug4158381() { | |
| Vector<String> sentenceSelectionData = new Vector<String>(); | |
| // Don't break sentence after period if it isn't followed by a space | |
| sentenceSelectionData.addElement("Test <code>Flags.Flag</code> class. "); | |
| sentenceSelectionData.addElement("Another test.\u2029"); | |
| // No breaks when there are no terminators around | |
| sentenceSelectionData.addElement("<P>Provides a set of " | |
| + ""lightweight" (all-java<FONT SIZE=\"-2\"><SUP>TM" | |
| + "</SUP></FONT> language) components that, " | |
| + "to the maximum degree possible, work the same on all platforms. "); | |
| sentenceSelectionData.addElement("Another test.\u2029"); | |
| generalIteratorTest(sentenceBreak, sentenceSelectionData); | |
| } | |
| @Test | |
| public void TestBug4143071() { | |
| Vector<String> sentenceSelectionData = new Vector<String>(); | |
| // Make sure sentences that end with digits work right | |
| sentenceSelectionData.addElement("Today is the 27th of May, 1998. "); | |
| sentenceSelectionData.addElement("Tomorrow with be 28 May 1998. "); | |
| sentenceSelectionData.addElement("The day after will be the 30th.\u2029"); | |
| generalIteratorTest(sentenceBreak, sentenceSelectionData); | |
| } | |
| @Test | |
| public void TestBug4152416() { | |
| Vector<String> sentenceSelectionData = new Vector<String>(); | |
| // Make sure sentences ending with a capital letter are treated correctly | |
| sentenceSelectionData.addElement("The type of all primitive " | |
| + "<code>boolean</code> values accessed in the target VM. "); | |
| sentenceSelectionData.addElement("Calls to xxx will return an " | |
| + "implementor of this interface.\u2029"); | |
| generalIteratorTest(sentenceBreak, sentenceSelectionData); | |
| } | |
| @Test | |
| public void TestBug4152117() { | |
| Vector<String> sentenceSelectionData = new Vector<String>(); | |
| // Make sure sentence breaking is handling punctuation correctly | |
| // [COULD NOT REPRODUCE THIS BUG, BUT TEST IS HERE TO MAKE SURE | |
| // IT DOESN'T CROP UP] | |
| sentenceSelectionData.addElement("Constructs a randomly generated " | |
| + "BigInteger, uniformly distributed over the range <tt>0</tt> " | |
| + "to <tt>(2<sup>numBits</sup> - 1)</tt>, inclusive. "); | |
| sentenceSelectionData.addElement("The uniformity of the distribution " | |
| + "assumes that a fair source of random bits is provided in " | |
| + "<tt>rnd</tt>. "); | |
| sentenceSelectionData.addElement("Note that this constructor always " | |
| + "constructs a non-negative BigInteger.\u2029"); | |
| generalIteratorTest(sentenceBreak, sentenceSelectionData); | |
| } | |
| @Test | |
| public void TestBug8264765() { | |
| Vector<String> sentenceSelectionData = new Vector<String>(); | |
| // Comma should not be regarded as the start of a sentence, | |
| // otherwise the backwards rule would break the following sentence. | |
| sentenceSelectionData.addElement( | |
| "Due to a problem (e.g., software bug), the server is down. "); | |
| generalIteratorTest(sentenceBreak, sentenceSelectionData); | |
| } | |
| @Test | |
| public void TestLineBreak() { | |
| Vector<String> lineSelectionData = new Vector<String>(); | |
| lineSelectionData.addElement("Multi-"); | |
| lineSelectionData.addElement("Level "); | |
| lineSelectionData.addElement("example "); | |
| lineSelectionData.addElement("of "); | |
| lineSelectionData.addElement("a "); | |
| lineSelectionData.addElement("semi-"); | |
| lineSelectionData.addElement("idiotic "); | |
| lineSelectionData.addElement("non-"); | |
| lineSelectionData.addElement("sensical "); | |
| lineSelectionData.addElement("(non-"); | |
| lineSelectionData.addElement("important) "); | |
| lineSelectionData.addElement("sentence. "); | |
| lineSelectionData.addElement("Hi "); | |
| lineSelectionData.addElement("Hello "); | |
| lineSelectionData.addElement("How\n"); | |
| lineSelectionData.addElement("are\r"); | |
| lineSelectionData.addElement("you\u2028"); | |
| lineSelectionData.addElement("fine.\t"); | |
| lineSelectionData.addElement("good. "); | |
| lineSelectionData.addElement("Now\r"); | |
| lineSelectionData.addElement("is\n"); | |
| lineSelectionData.addElement("the\r\n"); | |
| lineSelectionData.addElement("time\n"); | |
| lineSelectionData.addElement("\r"); | |
| lineSelectionData.addElement("for\r"); | |
| lineSelectionData.addElement("\r"); | |
| lineSelectionData.addElement("all"); | |
| generalIteratorTest(lineBreak, lineSelectionData); | |
| } | |
| @Test | |
| public void TestBug4068133() { | |
| Vector<String> lineSelectionData = new Vector<String>(); | |
| lineSelectionData.addElement("\u96f6"); | |
| lineSelectionData.addElement("\u4e00\u3002"); | |
| lineSelectionData.addElement("\u4e8c\u3001"); | |
| lineSelectionData.addElement("\u4e09\u3002\u3001"); | |
| lineSelectionData.addElement("\u56db\u3001\u3002\u3001"); | |
| lineSelectionData.addElement("\u4e94,"); | |
| lineSelectionData.addElement("\u516d."); | |
| lineSelectionData.addElement("\u4e03.\u3001,\u3002"); | |
| lineSelectionData.addElement("\u516b"); | |
| generalIteratorTest(lineBreak, lineSelectionData); | |
| } | |
| @Test | |
| public void TestBug4086052() { | |
| Vector<String> lineSelectionData = new Vector<String>(); | |
| lineSelectionData.addElement("foo\u00a0bar "); | |
| // lineSelectionData.addElement("foo\ufeffbar"); | |
| generalIteratorTest(lineBreak, lineSelectionData); | |
| } | |
| @Test | |
| public void TestBug4097920() { | |
| Vector<String> lineSelectionData = new Vector<String>(); | |
| lineSelectionData.addElement("dog,"); | |
| lineSelectionData.addElement("cat,"); | |
| lineSelectionData.addElement("mouse "); | |
| lineSelectionData.addElement("(one)"); | |
| lineSelectionData.addElement("(two)\n"); | |
| generalIteratorTest(lineBreak, lineSelectionData); | |
| } | |
| /* | |
| @Test | |
| public void TestBug4035266() { | |
| Vector<String> lineSelectionData = new Vector<String>(); | |
| lineSelectionData.addElement("The "); | |
| lineSelectionData.addElement("balance "); | |
| lineSelectionData.addElement("is "); | |
| lineSelectionData.addElement("$-23,456.78, "); | |
| lineSelectionData.addElement("not "); | |
| lineSelectionData.addElement("-$32,456.78!\n"); | |
| generalIteratorTest(lineBreak, lineSelectionData); | |
| } | |
| */ | |
| @Test | |
| public void TestBug4098467Lines() { | |
| Vector<String> lineSelectionData = new Vector<String>(); | |
| // What follows is a string of Korean characters (I found it in the Yellow Pages | |
| // ad for the Korean Presbyterian Church of San Francisco, and I hope I transcribed | |
| // it correctly), first as precomposed syllables, and then as conjoining jamo. | |
| // Both sequences should be semantically identical and break the same way. | |
| // precomposed syllables... | |
| lineSelectionData.addElement("\uc0c1"); | |
| lineSelectionData.addElement("\ud56d "); | |
| lineSelectionData.addElement("\ud55c"); | |
| lineSelectionData.addElement("\uc778 "); | |
| lineSelectionData.addElement("\uc5f0"); | |
| lineSelectionData.addElement("\ud569 "); | |
| lineSelectionData.addElement("\uc7a5"); | |
| lineSelectionData.addElement("\ub85c"); | |
| lineSelectionData.addElement("\uad50"); | |
| lineSelectionData.addElement("\ud68c "); | |
| // conjoining jamo... | |
| lineSelectionData.addElement("\u1109\u1161\u11bc\u1112\u1161\u11bc "); | |
| lineSelectionData.addElement("\u1112\u1161\u11ab\u110b\u1175\u11ab "); | |
| lineSelectionData.addElement("\u110b\u1167\u11ab\u1112\u1161\u11b8 "); | |
| lineSelectionData.addElement("\u110c\u1161\u11bc\u1105\u1169\u1100\u116d\u1112\u116c"); | |
| if (Locale.getDefault().getLanguage().equals("th")) { | |
| System.out.println("This test is skipped in th locale."); | |
| return; | |
| } | |
| generalIteratorTest(lineBreak, lineSelectionData); | |
| } | |
| @Test | |
| public void TestBug4117554Lines() { | |
| Vector<String> lineSelectionData = new Vector<String>(); | |
| // Fullwidth .!? should be treated as postJwrd | |
| lineSelectionData.addElement("\u4e01\uff0e"); | |
| lineSelectionData.addElement("\u4e02\uff01"); | |
| lineSelectionData.addElement("\u4e03\uff1f"); | |
| generalIteratorTest(lineBreak, lineSelectionData); | |
| } | |
| @Test | |
| public void TestBug4217703() { | |
| if (Locale.getDefault().getLanguage().equals("th")) { | |
| System.out.println("This test is skipped in th locale."); | |
| return; | |
| } | |
| Vector<String> lineSelectionData = new Vector<String>(); | |
| // There shouldn't be a line break between sentence-ending punctuation | |
| // and a closing quote | |
| lineSelectionData.addElement("He "); | |
| lineSelectionData.addElement("said "); | |
| lineSelectionData.addElement("\"Go!\" "); | |
| lineSelectionData.addElement("I "); | |
| lineSelectionData.addElement("went. "); | |
| lineSelectionData.addElement("Hashtable$Enumeration "); | |
| lineSelectionData.addElement("getText()."); | |
| lineSelectionData.addElement("getIndex()"); | |
| generalIteratorTest(lineBreak, lineSelectionData); | |
| } | |
| private static final String graveS = "S\u0300"; | |
| private static final String acuteBelowI = "i\u0317"; | |
| private static final String acuteE = "e\u0301"; | |
| private static final String circumflexA = "a\u0302"; | |
| private static final String tildeE = "e\u0303"; | |
| @Test | |
| public void TestCharacterBreak() { | |
| Vector<String> characterSelectionData = new Vector<String>(); | |
| characterSelectionData.addElement(graveS); | |
| characterSelectionData.addElement(acuteBelowI); | |
| characterSelectionData.addElement("m"); | |
| characterSelectionData.addElement("p"); | |
| characterSelectionData.addElement("l"); | |
| characterSelectionData.addElement(acuteE); | |
| characterSelectionData.addElement(" "); | |
| characterSelectionData.addElement("s"); | |
| characterSelectionData.addElement(circumflexA); | |
| characterSelectionData.addElement("m"); | |
| characterSelectionData.addElement("p"); | |
| characterSelectionData.addElement("l"); | |
| characterSelectionData.addElement(tildeE); | |
| characterSelectionData.addElement("."); | |
| characterSelectionData.addElement("w"); | |
| characterSelectionData.addElement(circumflexA); | |
| characterSelectionData.addElement("w"); | |
| characterSelectionData.addElement("a"); | |
| characterSelectionData.addElement("f"); | |
| characterSelectionData.addElement("q"); | |
| characterSelectionData.addElement("\n"); | |
| characterSelectionData.addElement("\r"); | |
| characterSelectionData.addElement("\r\n"); | |
| characterSelectionData.addElement("\n"); | |
| generalIteratorTest(characterBreak, characterSelectionData); | |
| } | |
| @Test | |
| public void TestBug4098467Characters() { | |
| Vector<String> characterSelectionData = new Vector<String>(); | |
| // What follows is a string of Korean characters (I found it in the Yellow Pages | |
| // ad for the Korean Presbyterian Church of San Francisco, and I hope I transcribed | |
| // it correctly), first as precomposed syllables, and then as conjoining jamo. | |
| // Both sequences should be semantically identical and break the same way. | |
| // precomposed syllables... | |
| characterSelectionData.addElement("\uc0c1"); | |
| characterSelectionData.addElement("\ud56d"); | |
| characterSelectionData.addElement(" "); | |
| characterSelectionData.addElement("\ud55c"); | |
| characterSelectionData.addElement("\uc778"); | |
| characterSelectionData.addElement(" "); | |
| characterSelectionData.addElement("\uc5f0"); | |
| characterSelectionData.addElement("\ud569"); | |
| characterSelectionData.addElement(" "); | |
| characterSelectionData.addElement("\uc7a5"); | |
| characterSelectionData.addElement("\ub85c"); | |
| characterSelectionData.addElement("\uad50"); | |
| characterSelectionData.addElement("\ud68c"); | |
| characterSelectionData.addElement(" "); | |
| // conjoining jamo... | |
| characterSelectionData.addElement("\u1109\u1161\u11bc"); | |
| characterSelectionData.addElement("\u1112\u1161\u11bc"); | |
| characterSelectionData.addElement(" "); | |
| characterSelectionData.addElement("\u1112\u1161\u11ab"); | |
| characterSelectionData.addElement("\u110b\u1175\u11ab"); | |
| characterSelectionData.addElement(" "); | |
| characterSelectionData.addElement("\u110b\u1167\u11ab"); | |
| characterSelectionData.addElement("\u1112\u1161\u11b8"); | |
| characterSelectionData.addElement(" "); | |
| characterSelectionData.addElement("\u110c\u1161\u11bc"); | |
| characterSelectionData.addElement("\u1105\u1169"); | |
| characterSelectionData.addElement("\u1100\u116d"); | |
| characterSelectionData.addElement("\u1112\u116c"); | |
| generalIteratorTest(characterBreak, characterSelectionData); | |
| } | |
| @Test | |
| public void TestBug4153072() { | |
| BreakIterator iter = BreakIterator.getWordInstance(); | |
| String str = "...Hello, World!..."; | |
| int begin = 3; | |
| int end = str.length() - 3; | |
| boolean gotException = false; | |
| boolean dummy; | |
| iter.setText(new StringCharacterIterator(str, begin, end, begin)); | |
| for (int index = -1; index < begin + 1; ++index) { | |
| try { | |
| dummy = iter.isBoundary(index); | |
| if (index < begin) | |
| fail("Didn't get exception with offset = " + index + | |
| " and begin index = " + begin); | |
| } | |
| catch (IllegalArgumentException e) { | |
| if (index >= begin) | |
| fail("Got exception with offset = " + index + | |
| " and begin index = " + begin); | |
| } | |
| } | |
| } | |
| @Test | |
| public void TestBug4146175Sentences() { | |
| Vector<String> sentenceSelectionData = new Vector<String>(); | |
| // break between periods and opening punctuation even when there's no | |
| // intervening space | |
| sentenceSelectionData.addElement("end."); | |
| sentenceSelectionData.addElement("(This is\u2029"); | |
| // treat the fullwidth period as an unambiguous sentence terminator | |
| sentenceSelectionData.addElement("\u7d42\u308f\u308a\uff0e"); | |
| sentenceSelectionData.addElement("\u300c\u3053\u308c\u306f"); | |
| generalIteratorTest(sentenceBreak, sentenceSelectionData); | |
| } | |
| @Test | |
| public void TestBug4146175Lines() { | |
| if (Locale.getDefault().getLanguage().equals("th")) { | |
| System.out.println("This test is skipped in th locale."); | |
| return; | |
| } | |
| Vector<String> lineSelectionData = new Vector<String>(); | |
| // the fullwidth comma should stick to the preceding Japanese character | |
| lineSelectionData.addElement("\u7d42\uff0c"); | |
| lineSelectionData.addElement("\u308f"); | |
| generalIteratorTest(lineBreak, lineSelectionData); | |
| } | |
| @Test | |
| public void TestBug4214367() { | |
| if (Locale.getDefault().getLanguage().equals("th")) { | |
| System.out.println("This test is skipped in th locale."); | |
| return; | |
| } | |
| Vector<String> wordSelectionData = new Vector<String>(); | |
| // the hiragana and katakana iteration marks and the long vowel mark | |
| // are not being treated correctly by the word-break iterator | |
| wordSelectionData.addElement("\u3042\u3044\u309d\u3042\u309e\u3042\u30fc\u3042"); | |
| wordSelectionData.addElement("\u30a2\u30a4\u30fd\u30a2\u30fe\u30a2\u30fc\u30a2"); | |
| generalIteratorTest(wordBreak, wordSelectionData); | |
| } | |
| private static final String cannedTestChars // characters fo the class Cc are ignorable for breaking | |
| = /*"\u0000\u0001\u0002\u0003\u0004*/" !\"#$%&()+-01234<=>ABCDE[]^_`abcde{}|\u00a0\u00a2" | |
| + "\u00a3\u00a4\u00a5\u00a6\u00a7\u00a8\u00a9\u00ab\u00ad\u00ae\u00af\u00b0\u00b2\u00b3" | |
| + "\u00b4\u00b9\u00bb\u00bc\u00bd\u02b0\u02b1\u02b2\u02b3\u02b4\u0300\u0301\u0302\u0303" | |
| + "\u0304\u05d0\u05d1\u05d2\u05d3\u05d4\u0903\u093e\u093f\u0940\u0949\u0f3a\u0f3b\u2000" | |
| + "\u2001\u2002\u200c\u200d\u200e\u200f\u2010\u2011\u2012\u2028\u2029\u202a\u203e\u203f" | |
| + "\u2040\u20dd\u20de\u20df\u20e0\u2160\u2161\u2162\u2163\u2164"; | |
| @Test | |
| public void TestSentenceInvariants() | |
| { | |
| BreakIterator e = BreakIterator.getSentenceInstance(); | |
| doOtherInvariantTest(e, cannedTestChars + ".,\u3001\u3002\u3041\u3042\u3043\ufeff"); | |
| } | |
| @Test | |
| public void TestWordInvariants() | |
| { | |
| if (Locale.getDefault().getLanguage().equals("th")) { | |
| System.out.println("This test is skipped in th locale."); | |
| return; | |
| } | |
| BreakIterator e = BreakIterator.getWordInstance(); | |
| doBreakInvariantTest(e, cannedTestChars + "\',.\u3041\u3042\u3043\u309b\u309c\u30a1\u30a2" | |
| + "\u30a3\u4e00\u4e01\u4e02"); | |
| doOtherInvariantTest(e, cannedTestChars + "\',.\u3041\u3042\u3043\u309b\u309c\u30a1\u30a2" | |
| + "\u30a3\u4e00\u4e01\u4e02"); | |
| } | |
| @Test | |
| public void TestLineInvariants() | |
| { | |
| if (Locale.getDefault().getLanguage().equals("th")) { | |
| System.out.println("This test is skipped in th locale."); | |
| return; | |
| } | |
| BreakIterator e = BreakIterator.getLineInstance(); | |
| String testChars = cannedTestChars + ".,;:\u3001\u3002\u3041\u3042\u3043\u3044\u3045" | |
| + "\u30a3\u4e00\u4e01\u4e02"; | |
| doBreakInvariantTest(e, testChars); | |
| doOtherInvariantTest(e, testChars); | |
| int errorCount = 0; | |
| // in addition to the other invariants, a line-break iterator should make sure that: | |
| // it doesn't break around the non-breaking characters | |
| String noBreak = "\u00a0\u2007\u2011\ufeff"; | |
| StringBuffer work = new StringBuffer("aaa"); | |
| for (int i = 0; i < testChars.length(); i++) { | |
| char c = testChars.charAt(i); | |
| if (c == '\r' || c == '\n' || c == '\u2029' || c == '\u2028' || c == '\u0003') | |
| continue; | |
| work.setCharAt(0, c); | |
| for (int j = 0; j < noBreak.length(); j++) { | |
| work.setCharAt(1, noBreak.charAt(j)); | |
| for (int k = 0; k < testChars.length(); k++) { | |
| work.setCharAt(2, testChars.charAt(k)); | |
| // CONTROL (Cc) and FORMAT (Cf) Characters are to be ignored | |
| // for breaking purposes as per UTR14 | |
| int type1 = Character.getType(work.charAt(1)); | |
| int type2 = Character.getType(work.charAt(2)); | |
| if (type1 == Character.CONTROL || type1 == Character.FORMAT || | |
| type2 == Character.CONTROL || type2 == Character.FORMAT) { | |
| continue; | |
| } | |
| e.setText(work.toString()); | |
| for (int l = e.first(); l != BreakIterator.DONE; l = e.next()) { | |
| if (l == 1 || l == 2) { | |
| //errln("Got break between U+" + Integer.toHexString((int) | |
| // (work.charAt(l - 1))) + " and U+" + Integer.toHexString( | |
| // (int)(work.charAt(l))) + "\ntype1 = " + type1 + "\ntype2 = " + type2); | |
| // as per UTR14 spaces followed by a GLUE character should allow | |
| // line breaking | |
| if (work.charAt(l-1) == '\u0020' && (work.charAt(l) == '\u00a0' || | |
| work.charAt(l) == '\u0f0c' || | |
| work.charAt(l) == '\u2007' || | |
| work.charAt(l) == '\u2011' || | |
| work.charAt(l) == '\u202f' || | |
| work.charAt(l) == '\ufeff')) { | |
| continue; | |
| } | |
| fail("Got break between U+" + Integer.toHexString((int) | |
| (work.charAt(l - 1))) + " and U+" + Integer.toHexString( | |
| (int)(work.charAt(l)))); | |
| errorCount++; | |
| if (errorCount >= 75) | |
| return; | |
| } | |
| } | |
| } | |
| } | |
| } | |
| // The following test has so many exceptions that it would be better to write a new set of data | |
| // that tested exactly what should be tested | |
| // Until that point it will be commented out | |
| /* | |
| // it does break after dashes (unless they're followed by a digit, a non-spacing mark, | |
| // a currency symbol, a space, a format-control character, a regular control character, | |
| // a line or paragraph separator, or another dash) | |
| String dashes = "-\u00ad\u2010\u2012\u2013\u2014"; | |
| for (int i = 0; i < testChars.length(); i++) { | |
| work.setCharAt(0, testChars.charAt(i)); | |
| for (int j = 0; j < dashes.length(); j++) { | |
| work.setCharAt(1, dashes.charAt(j)); | |
| for (int k = 0; k < testChars.length(); k++) { | |
| char c = testChars.charAt(k); | |
| if (Character.getType(c) == Character.DECIMAL_DIGIT_NUMBER || | |
| Character.getType(c) == Character.OTHER_NUMBER || | |
| Character.getType(c) == Character.NON_SPACING_MARK || | |
| Character.getType(c) == Character.ENCLOSING_MARK || | |
| Character.getType(c) == Character.CURRENCY_SYMBOL || | |
| Character.getType(c) == Character.DASH_PUNCTUATION || | |
| Character.getType(c) == Character.SPACE_SEPARATOR || | |
| Character.getType(c) == Character.FORMAT || | |
| Character.getType(c) == Character.CONTROL || | |
| Character.getType(c) == Character.END_PUNCTUATION || | |
| Character.getType(c) == Character.FINAL_QUOTE_PUNCTUATION || | |
| Character.getType(c) == Character.OTHER_PUNCTUATION || | |
| c == '\'' || c == '\"' || | |
| // category EX as per UTR14 | |
| c == '!' || c == '?' || c == '\ufe56' || c == '\ufe57' || c == '\uff01' || c == '\uff1f' || | |
| c == '\n' || c == '\r' || c == '\u2028' || c == '\u2029' || | |
| c == '\u0003' || c == '\u2007' || c == '\u2011' || | |
| c == '\ufeff') | |
| continue; | |
| work.setCharAt(2, c); | |
| e.setText(work.toString()); | |
| boolean saw2 = false; | |
| for (int l = e.first(); l != BreakIterator.DONE; l = e.next()) | |
| if (l == 2) | |
| saw2 = true; | |
| if (!saw2) { | |
| fail("Didn't get break between U+" + Integer.toHexString((int) | |
| (work.charAt(1))) + " and U+" + Integer.toHexString( | |
| (int)(work.charAt(2)))); | |
| errorCount++; | |
| if (errorCount >= 75) | |
| return; | |
| } | |
| } | |
| } | |
| } | |
| */ | |
| } | |
| @Test | |
| public void TestCharacterInvariants() | |
| { | |
| BreakIterator e = BreakIterator.getCharacterInstance(); | |
| doBreakInvariantTest(e, cannedTestChars + "\u1100\u1101\u1102\u1160\u1161\u1162\u11a8" | |
| + "\u11a9\u11aa"); | |
| doOtherInvariantTest(e, cannedTestChars + "\u1100\u1101\u1102\u1160\u1161\u1162\u11a8" | |
| + "\u11a9\u11aa"); | |
| } | |
| @Test | |
| public void TestEmptyString() | |
| { | |
| String text = ""; | |
| Vector<String> x = new Vector<String>(); | |
| x.addElement(text); | |
| generalIteratorTest(lineBreak, x); | |
| } | |
| @Test | |
| public void TestGetAvailableLocales() | |
| { | |
| Locale[] locList = BreakIterator.getAvailableLocales(); | |
| if (locList.length == 0) | |
| fail("getAvailableLocales() returned an empty list!"); | |
| // I have no idea how to test this function... | |
| } | |
| /** | |
| * Bug 4095322 | |
| */ | |
| @Test | |
| public void TestJapaneseLineBreak() | |
| { | |
| StringBuffer testString = new StringBuffer("\u4e00x\u4e8c"); | |
| // Breaking on <Kanji>$<Kanji> is inconsistent | |
| /* Characters in precedingChars and followingChars have been updated | |
| * from Unicode 2.0.14-based to 3.0.0-based when 4638433 was fixed. | |
| * In concrete terms, | |
| * 0x301F : Its category was changed from Ps to Pe since Unicode 2.1. | |
| * 0x169B & 0x169C : added since Unicode 3.0.0. | |
| */ | |
| String precedingChars = | |
| /* Puctuation, Open */ | |
| "([{\u201a\u201e\u2045\u207d\u208d\u2329\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u301d\ufe35\ufe37\ufe39\ufe3b\ufe3d\ufe3f\ufe41\ufe43\ufe59\ufe5b\ufe5d\uff08\uff3b\uff5b\uff62\u169b" | |
| /* Punctuation, Initial quote */ | |
| + "\u00ab\u2018\u201b\u201c\u201f\u2039" | |
| /* Symbol, Currency */ | |
| + "\u00a5\u00a3\u00a4\u20a0"; | |
| String followingChars = | |
| /* Puctuation, Close */ | |
| ")]}\u2046\u207e\u208e\u232a\u3009\u300b\u300d\u300f\u3011\u3015\u3017\u3019\u301b\u301e\u301f\ufd3e\ufe36\ufe38\ufe3a\ufe3c\ufe3e\ufe40\ufe42\ufe44\ufe5a\ufe5c\ufe5e\uff09\uff3d\uff5d\uff63\u169c" | |
| /* Punctuation, Final quote */ | |
| + "\u00bb\u2019\u201d\u203a" | |
| /* Punctuation, Other */ | |
| + "!%,.:;\u3001\u3002\u2030\u2031\u2032\u2033\u2034" | |
| /* Punctuation, Dash */ | |
| + "\u2103\u2109" | |
| /* Symbol, Currency */ | |
| + "\u00a2" | |
| /* Letter, Modifier */ | |
| + "\u3005\u309d\u309e" | |
| /* Letter, Other */ | |
| + "\u3063\u3083\u3085\u3087\u30c3\u30e3\u30e5\u30e7\u30fc\u30fd\u30fe" | |
| /* Mark, Non-Spacing */ | |
| + "\u0300\u0301\u0302" | |
| /* Symbol, Modifier */ | |
| + "\u309b\u309c" | |
| /* Symbol, Other */ | |
| + "\u00b0"; | |
| BreakIterator iter = BreakIterator.getLineInstance(Locale.JAPAN); | |
| for (int i = 0; i < precedingChars.length(); i++) { | |
| testString.setCharAt(1, precedingChars.charAt(i)); | |
| iter.setText(testString.toString()); | |
| int j = iter.first(); | |
| if (j != 0) { | |
| fail("ja line break failure: failed to start at 0 and bounced at " + j); | |
| } | |
| j = iter.next(); | |
| if (j != 1) { | |
| fail("ja line break failure: failed to stop before '" | |
| + precedingChars.charAt(i) + "' (\\u" | |
| + Integer.toString(precedingChars.charAt(i), 16) | |
| + ") at 1 and bounded at " + j); | |
| } | |
| j = iter.next(); | |
| if (j != 3) { | |
| fail("ja line break failure: failed to skip position after '" | |
| + precedingChars.charAt(i) + "' (\\u" | |
| + Integer.toString(precedingChars.charAt(i), 16) | |
| + ") at 3 and bounded at " + j); | |
| } | |
| } | |
| for (int i = 0; i < followingChars.length(); i++) { | |
| testString.setCharAt(1, followingChars.charAt(i)); | |
| iter.setText(testString.toString()); | |
| int j = iter.first(); | |
| if (j != 0) { | |
| fail("ja line break failure: failed to start at 0 and bounded at " + j); | |
| } | |
| j = iter.next(); | |
| if (j != 2) { | |
| fail("ja line break failure: failed to skip position before '" | |
| + followingChars.charAt(i) + "' (\\u" | |
| + Integer.toString(followingChars.charAt(i), 16) | |
| + ") at 2 and bounded at " + j); | |
| } | |
| j = iter.next(); | |
| if (j != 3) { | |
| fail("ja line break failure: failed to stop after '" | |
| + followingChars.charAt(i) + "' (\\u" | |
| + Integer.toString(followingChars.charAt(i), 16) | |
| + ") at 3 and bounded at " + j); | |
| } | |
| } | |
| } | |
| /** | |
| * Bug 4638433 | |
| */ | |
| @Test | |
| public void TestLineBreakBasedOnUnicode3_0_0() | |
| { | |
| BreakIterator iter; | |
| int i; | |
| /* Latin Extend-B characters | |
| * 0x0218-0x0233 which have been added since Unicode 3.0.0. | |
| */ | |
| iter = BreakIterator.getWordInstance(Locale.US); | |
| iter.setText("\u0216\u0217\u0218\u0219\u021A"); | |
| i = iter.first(); | |
| i = iter.next(); | |
| if (i != 5) { | |
| fail("Word break failure: failed to stop at 5 and bounded at " + i); | |
| } | |
| iter = BreakIterator.getLineInstance(Locale.US); | |
| /* <Three(Nd)><Two(Nd)><Low Double Prime Quotation Mark(Pe)><One(Nd)> | |
| * \u301f has changed its category from Ps to Pe since Unicode 2.1. | |
| */ | |
| iter.setText("32\u301f1"); | |
| i = iter.first(); | |
| i = iter.next(); | |
| if (i != 3) { | |
| fail("Line break failure: failed to skip before \\u301F(Pe) at 3 and bounded at " + i); | |
| } | |
| /* Mongolian <Letter A(Lo)><Todo Soft Hyphen(Pd)><Letter E(Lo)> | |
| * which have been added since Unicode 3.0.0. | |
| */ | |
| iter.setText("\u1820\u1806\u1821"); | |
| i = iter.first(); | |
| i = iter.next(); | |
| if (i != 2) { | |
| fail("Mongolian line break failure: failed to skip position before \\u1806(Pd) at 2 and bounded at " + i); | |
| } | |
| /* Khmer <ZERO(Nd)><Currency Symbol(Sc)><ONE(Nd)> which have | |
| * been added since Unicode 3.0.0. | |
| */ | |
| iter.setText("\u17E0\u17DB\u17E1"); | |
| i = iter.first(); | |
| i = iter.next(); | |
| if (i != 1) { | |
| fail("Khmer line break failure: failed to stop before \\u17DB(Sc) at 1 and bounded at " + i); | |
| } | |
| i = iter.next(); | |
| if (i != 3) { | |
| fail("Khmer line break failure: failed to skip position after \\u17DB(Sc) at 3 and bounded at " + i); | |
| } | |
| /* Ogham <Letter UR(Lo)><Space Mark(Zs)><Letter OR(Lo)> which have | |
| * been added since Unicode 3.0.0. | |
| */ | |
| iter.setText("\u1692\u1680\u1696"); | |
| i = iter.first(); | |
| i = iter.next(); | |
| if (i != 2) { | |
| fail("Ogham line break failure: failed to skip postion before \\u1680(Zs) at 2 and bounded at " + i); | |
| } | |
| // Confirm changes in BreakIteratorRules_th.java have been reflected. | |
| iter = BreakIterator.getLineInstance(Locale.of("th")); | |
| /* Thai <Seven(Nd)> | |
| * <Left Double Quotation Mark(Pi)> | |
| * <Five(Nd)> | |
| * <Right Double Quotation Mark(Pf)> | |
| * <Three(Nd)> | |
| */ | |
| iter.setText("\u0E57\u201C\u0E55\u201D\u0E53"); | |
| i = iter.first(); | |
| i = iter.next(); | |
| if (i != 1) { | |
| fail("Thai line break failure: failed to stop before \\u201C(Pi) at 1 and bounded at " + i); | |
| } | |
| i = iter.next(); | |
| if (i != 4) { | |
| fail("Thai line break failure: failed to stop after \\u201D(Pf) at 4 and bounded at " + i); | |
| } | |
| } | |
| /** | |
| * Bug 4068137 | |
| */ | |
| @Test | |
| public void TestEndBehavior() | |
| { | |
| String testString = "boo."; | |
| BreakIterator wb = BreakIterator.getWordInstance(); | |
| wb.setText(testString); | |
| if (wb.first() != 0) | |
| fail("Didn't get break at beginning of string."); | |
| if (wb.next() != 3) | |
| fail("Didn't get break before period in \"boo.\""); | |
| if (wb.current() != 4 && wb.next() != 4) | |
| fail("Didn't get break at end of string."); | |
| } | |
| // [serialization test has been removed pursuant to bug #4152965] | |
| /** | |
| * Bug 4450804 | |
| */ | |
| @Test | |
| public void TestLineBreakContractions() { | |
| Vector<String> expected = new Vector<String>(); | |
| expected.add("These "); | |
| expected.add("are "); | |
| expected.add("'foobles'. "); | |
| expected.add("Don't "); | |
| expected.add("you "); | |
| expected.add("like "); | |
| expected.add("them?"); | |
| generalIteratorTest(lineBreak, expected); | |
| } | |
| private static final Pattern CODEPOINT = Pattern.compile("([0-9A-F]{4,5})"); | |
| @Test | |
| public void TestGraphemeBreak() throws Exception { | |
| Files.lines(Paths.get(System.getProperty("test.root"), | |
| "../../src/java.base/share/data/unicodedata/auxiliary/GraphemeBreakTest.txt")) | |
| .map(ln -> ln.replaceFirst("#.*", "")) | |
| .filter(Predicate.not(String::isEmpty)) | |
| .map(line -> line.split("\\s*÷[\\s\\t]*")) | |
| .forEach(sa -> { | |
| Vector<String> expected = new Vector<>( | |
| Arrays.stream(sa) | |
| .map(line -> CODEPOINT.matcher(line).replaceAll(mr -> Character.toString(Integer.valueOf(mr.group(),16)))) | |
| .map(line -> line.replaceAll("\\s×\\s", "")) | |
| .filter(Predicate.not(String::isEmpty)) | |
| .toList()); | |
| generalIteratorTest(characterBreak, expected); | |
| }); | |
| } | |
| @Test | |
| public void TestSetTextIOOBException() { | |
| BreakIterator.getCharacterInstance().setText(new StringCharacterIterator("abcfefg", 1, 5, 3)); | |
| } | |
| } |