| //package org.codehaus.jackson.io; | |
| import java.io.*; | |
| import org.codehaus.jackson.io.IOContext; | |
| /** | |
| * Optimized Reader that reads UTF-8 encoded content from an input stream. | |
| * In addition to doing (hopefully) optimal conversion, it can also take | |
| * array of "pre-read" (leftover) bytes; this is necessary when preliminary | |
| * stream/reader is trying to figure out character encoding. | |
| */ | |
| public final class UTF8Reader | |
| extends BaseReader | |
| { | |
| char mSurrogate = NULL_CHAR; | |
| /** | |
| * Total read character count; used for error reporting purposes | |
| */ | |
| int mCharCount = 0; | |
| /** | |
| * Total read byte count; used for error reporting purposes | |
| */ | |
| int mByteCount = 0; | |
| /* | |
| //////////////////////////////////////// | |
| // Life-cycle | |
| //////////////////////////////////////// | |
| */ | |
| public UTF8Reader(IOContext ctxt, | |
| InputStream in, byte[] buf, int ptr, int len) | |
| { | |
| super(ctxt, in, buf, ptr, len); | |
| } | |
| /* | |
| //////////////////////////////////////// | |
| // Public API | |
| //////////////////////////////////////// | |
| */ | |
| public int read(char[] cbuf, int start, int len) | |
| throws IOException | |
| { | |
| // Already EOF? | |
| if (mBuffer == null) { | |
| return -1; | |
| } | |
| if (len < 1) { | |
| return len; | |
| } | |
| // Let's then ensure there's enough room... | |
| if (start < 0 || (start+len) > cbuf.length) { | |
| reportBounds(cbuf, start, len); | |
| } | |
| len += start; | |
| int outPtr = start; | |
| // Ok, first; do we have a surrogate from last round? | |
| if (mSurrogate != NULL_CHAR) { | |
| cbuf[outPtr++] = mSurrogate; | |
| mSurrogate = NULL_CHAR; | |
| // No need to load more, already got one char | |
| } else { | |
| /* To prevent unnecessary blocking (esp. with network streams), | |
| * we'll only require decoding of a single char | |
| */ | |
| int left = (mLength - mPtr); | |
| /* So; only need to load more if we can't provide at least | |
| * one more character. We need not do thorough check here, | |
| * but let's check the common cases here: either completely | |
| * empty buffer (left == 0), or one with less than max. byte | |
| * count for a single char, and starting of a multi-byte | |
| * encoding (this leaves possibility of a 2/3-byte char | |
| * that is still fully accessible... but that can be checked | |
| * by the load method) | |
| */ | |
| if (left < 4) { | |
| // Need to load more? | |
| if (left < 1 || mBuffer[mPtr] < 0) { | |
| if (!loadMore(left)) { // (legal) EOF? | |
| return -1; | |
| } | |
| } | |
| } | |
| } | |
| /* This may look silly, but using a local var is indeed faster | |
| * (if and when HotSpot properly gets things running) than | |
| * member variable... | |
| */ | |
| byte[] buf = mBuffer; | |
| int inPtr = mPtr; | |
| int inBufLen = mLength; | |
| main_loop: | |
| while (outPtr < len) { | |
| // At this point we have at least one byte available | |
| int c = (int) buf[inPtr++]; | |
| /* Let's first do the quickie loop for common case; 7-bit | |
| * ascii: | |
| */ | |
| if (c >= 0) { // ascii? can probably loop, then | |
| cbuf[outPtr++] = (char) c; // ok since MSB is never on | |
| /* Ok, how many such chars could we safely process | |
| * without overruns? (will combine 2 in-loop comparisons | |
| * into just one) | |
| */ | |
| int outMax = (len - outPtr); // max output | |
| int inMax = (inBufLen - inPtr); // max input | |
| int inEnd = inPtr + ((inMax < outMax) ? inMax : outMax); | |
| ascii_loop: | |
| while (true) { | |
| if (inPtr >= inEnd) { | |
| break main_loop; | |
| } | |
| c = (int) buf[inPtr++]; | |
| if (c < 0) { // multi-byte | |
| break ascii_loop; | |
| } | |
| cbuf[outPtr++] = (char) c; | |
| } | |
| } | |
| int needed; | |
| // Ok; if we end here, we got multi-byte combination | |
| if ((c & 0xE0) == 0xC0) { // 2 bytes (0x0080 - 0x07FF) | |
| c = (c & 0x1F); | |
| needed = 1; | |
| } else if ((c & 0xF0) == 0xE0) { // 3 bytes (0x0800 - 0xFFFF) | |
| c = (c & 0x0F); | |
| needed = 2; | |
| } else if ((c & 0xF8) == 0xF0) { | |
| // 4 bytes; double-char BS, with surrogates and all... | |
| c = (c & 0x0F); | |
| needed = 3; | |
| } else { | |
| reportInvalidInitial(c & 0xFF, outPtr-start); | |
| // never gets here... | |
| needed = 1; | |
| } | |
| /* Do we have enough bytes? If not, let's just push back the | |
| * byte and leave, since we have already gotten at least one | |
| * char decoded. This way we will only block (with read from | |
| * input stream) when absolutely necessary. | |
| */ | |
| if ((inBufLen - inPtr) < needed) { | |
| --inPtr; | |
| break main_loop; | |
| } | |
| int d = (int) buf[inPtr++]; | |
| if ((d & 0xC0) != 0x080) { | |
| reportInvalidOther(d & 0xFF, outPtr-start); | |
| } | |
| c = (c << 6) | (d & 0x3F); | |
| if (needed > 1) { // needed == 1 means 2 bytes total | |
| d = buf[inPtr++]; // 3rd byte | |
| if ((d & 0xC0) != 0x080) { | |
| reportInvalidOther(d & 0xFF, outPtr-start); | |
| } | |
| c = (c << 6) | (d & 0x3F); | |
| if (needed > 2) { // 4 bytes? (need surrogates) | |
| d = buf[inPtr++]; | |
| if ((d & 0xC0) != 0x080) { | |
| reportInvalidOther(d & 0xFF, outPtr-start); | |
| } | |
| c = (c << 6) | (d & 0x3F); | |
| if (c > LAST_VALID_UNICODE_CHAR) { | |
| reportInvalid(c, outPtr-start, | |
| "(above "+Integer.toHexString(LAST_VALID_UNICODE_CHAR)); | |
| } | |
| /* Ugh. Need to mess with surrogates. Ok; let's inline them | |
| * there, then, if there's room: if only room for one, | |
| * need to save the surrogate for the rainy day... | |
| */ | |
| c -= 0x10000; // to normalize it starting with 0x0 | |
| cbuf[outPtr++] = (char) (0xD800 + (c >> 10)); | |
| // hmmh. can this ever be 0? (not legal, at least?) | |
| c = (0xDC00 | (c & 0x03FF)); | |
| // Room for second part? | |
| if (outPtr >= len) { // nope | |
| mSurrogate = (char) c; | |
| break main_loop; | |
| } | |
| // sure, let's fall back to normal processing: | |
| } | |
| /* 08-Jun-2007, TSa: Not sure if it's really legal | |
| * to get surrogate chars here: JSON specs do not | |
| * prevent them, which is different from xml. So | |
| * for now let's not worry about them. If checks | |
| * are needed, can uncomment following: | |
| */ | |
| /* | |
| else { | |
| // Otherwise, need to check that 3-byte chars are | |
| // legal ones (should not expand to surrogates) | |
| if (c >= 0xD800) { | |
| // But first, let's check max chars: | |
| if (c < 0xE000) { | |
| reportInvalid(c, outPtr-start, "(a surrogate character) "); | |
| } | |
| } | |
| } | |
| */ | |
| } | |
| cbuf[outPtr++] = (char) c; | |
| if (inPtr >= inBufLen) { | |
| break main_loop; | |
| } | |
| } | |
| mPtr = inPtr; | |
| len = outPtr - start; | |
| mCharCount += len; | |
| return len; | |
| } | |
| /* | |
| //////////////////////////////////////// | |
| // Internal methods | |
| //////////////////////////////////////// | |
| */ | |
| private void reportInvalidInitial(int mask, int offset) | |
| throws IOException | |
| { | |
| // input (byte) ptr has been advanced by one, by now: | |
| int bytePos = mByteCount + mPtr - 1; | |
| int charPos = mCharCount + offset + 1; | |
| throw new CharConversionException("Invalid UTF-8 start byte 0x" | |
| +Integer.toHexString(mask) | |
| +" (at char #"+charPos+", byte #"+bytePos+")"); | |
| } | |
| private void reportInvalidOther(int mask, int offset) | |
| throws IOException | |
| { | |
| int bytePos = mByteCount + mPtr - 1; | |
| int charPos = mCharCount + offset; | |
| throw new CharConversionException("Invalid UTF-8 middle byte 0x" | |
| +Integer.toHexString(mask) | |
| +" (at char #"+charPos+", byte #"+bytePos+")"); | |
| } | |
| private void reportUnexpectedEOF(int gotBytes, int needed) | |
| throws IOException | |
| { | |
| int bytePos = mByteCount + gotBytes; | |
| int charPos = mCharCount; | |
| throw new CharConversionException("Unexpected EOF in the middle of a multi-byte char: got " | |
| +gotBytes+", needed "+needed | |
| +", at char #"+charPos+", byte #"+bytePos+")"); | |
| } | |
| private void reportInvalid(int value, int offset, String msg) | |
| throws IOException | |
| { | |
| int bytePos = mByteCount + mPtr - 1; | |
| int charPos = mCharCount + offset; | |
| throw new CharConversionException("Invalid UTF-8 character 0x" | |
| +Integer.toHexString(value)+msg | |
| +" at char #"+charPos+", byte #"+bytePos+")"); | |
| } | |
| /** | |
| * @param available Number of "unused" bytes in the input buffer | |
| * | |
| * @return True, if enough bytes were read to allow decoding of at least | |
| * one full character; false if EOF was encountered instead. | |
| */ | |
| private boolean loadMore(int available) | |
| throws IOException | |
| { | |
| mByteCount += (mLength - available); | |
| // Bytes that need to be moved to the beginning of buffer? | |
| if (available > 0) { | |
| if (mPtr > 0) { | |
| for (int i = 0; i < available; ++i) { | |
| mBuffer[i] = mBuffer[mPtr+i]; | |
| } | |
| mPtr = 0; | |
| } | |
| mLength = available; | |
| } else { | |
| /* Ok; here we can actually reasonably expect an EOF, | |
| * so let's do a separate read right away: | |
| */ | |
| mPtr = 0; | |
| int count = mIn.read(mBuffer); | |
| if (count < 1) { | |
| mLength = 0; | |
| if (count < 0) { // -1 | |
| freeBuffers(); // to help GC? | |
| return false; | |
| } | |
| // 0 count is no good; let's err out | |
| reportStrangeStream(); | |
| } | |
| mLength = count; | |
| } | |
| /* We now have at least one byte... and that allows us to | |
| * calculate exactly how many bytes we need! | |
| */ | |
| int c = (int) mBuffer[0]; | |
| if (c >= 0) { // single byte (ascii) char... cool, can return | |
| return true; | |
| } | |
| // Ok, a multi-byte char, let's check how many bytes we'll need: | |
| int needed; | |
| if ((c & 0xE0) == 0xC0) { // 2 bytes (0x0080 - 0x07FF) | |
| needed = 2; | |
| } else if ((c & 0xF0) == 0xE0) { // 3 bytes (0x0800 - 0xFFFF) | |
| needed = 3; | |
| } else if ((c & 0xF8) == 0xF0) { | |
| // 4 bytes; double-char BS, with surrogates and all... | |
| needed = 4; | |
| } else { | |
| reportInvalidInitial(c & 0xFF, 0); | |
| // never gets here... but compiler whines without this: | |
| needed = 1; | |
| } | |
| /* And then we'll just need to load up to that many bytes; | |
| * if an EOF is hit, that'll be an error. But we need not do | |
| * actual decoding here, just load enough bytes. | |
| */ | |
| while (mLength < needed) { | |
| int count = mIn.read(mBuffer, mLength, mBuffer.length - mLength); | |
| if (count < 1) { | |
| if (count < 0) { // -1, EOF... no good! | |
| freeBuffers(); | |
| reportUnexpectedEOF(mLength, needed); | |
| } | |
| // 0 count is no good; let's err out | |
| reportStrangeStream(); | |
| } | |
| mLength += count; | |
| } | |
| return true; | |
| } | |
| } | |