blob: 55181509c9dcaa46aea1c774bb2aa6459b593a68 [file] [log] [blame]
///////////////////////////////////////////////////////////////////////////////
//
// The contents of this file are subject to the Mozilla Public License
// Version 1.1 (the "License"); you may not use this file except in
// compliance with the License. You may obtain a copy of the License at
// http://www.mozilla.org/MPL/
//
// Software distributed under the License is distributed on an "AS IS"
// basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
// License for the specific language governing rights and limitations
// under the License.
//
// The Original Code is MP4v2.
//
// The Initial Developer of the Original Code is David Byron.
// Portions created by David Byron are Copyright (C) 2010.
// All Rights Reserved.
//
// Contributors:
// David Byron, dbyron@dbyron.com
//
///////////////////////////////////////////////////////////////////////////////
#include "src/impl.h"
#include "libplatform/impl.h" /* for platform_win32_impl.h which declares Utf8ToFilename */
#include <algorithm> /* for replace */
#include <windows.h>
namespace mp4v2 {
using namespace impl;
}
/**
* Set this to 1 to compile in extra debugging
*/
#define EXTRA_DEBUG 0
/**
* @def LOG_PRINTF
*
* call log.printf if EXTRA_DEBUG is defined to 1. Do
* nothing otherwise
*/
#if EXTRA_DEBUG
#define LOG_PRINTF(X) log.printf X
#else
#define LOG_PRINTF(X)
#endif
/**
* Section 2.13 "Special Characters and Noncharacters" of
* _The Unicode Standard, Version 5.0_
* (http://www.unicode.org/versions/Unicode5.0.0/bookmarks.html)
* defines "The Replacement Character" U+FFFD as the
* "general substitute character" that "can be substituted
* for any 'unknown' character in another encoding that can
* not be mapped in terms of known Unicode characters"
*
* See also section D.7 of 10646.
*/
#define REPLACEMENT_CHAR 0xFFFD
namespace mp4v2 { namespace platform { namespace win32 {
/**
* A structure to store the number of characters required to
* encode a particular UCS-4 character in UTF-8
*/
struct utf8_len_info
{
/**
* This structure applies to a number >= @p range_min.
*/
UINT32 range_min;
/**
* This structure applies to a number <= @p range_max.
*/
UINT32 range_max;
/**
* The number of characters required to encode a number
* in [@p range_min,@p range_max] as UTF-8.
*/
size_t num_chars;
};
/**
* A structure to store the number of characters required to
* encode a particular UCS-4 character in UTF-8. For now
* we're using wide characters (which according to
* http://msdn.microsoft.com/en-us/library/ms776414.aspx
* means UTF-16 since Windows 2000) so we're only using up
* to 4-byte UTF-8 sequences. Parts of the range aren't
* valid (e.g. [U+D800,U+DFFF] but that's handled elsewhere.
*/
static struct utf8_len_info s_len_info[] =
{
{ 0x00000000, 0x0000007F, 1 },
{ 0x00000080, 0x000007FF, 2 },
{ 0x00000800, 0x0000FFFF, 3 },
{ 0x00010000, 0x001FFFFF, 4 },
{ 0x00200000, 0x03FFFFFF, 5 },
{ 0x04000000, 0x7FFFFFFF, 6 }
};
/**
* Utf8ToFilename constructor
*
* @param utf8string a UTF-8 encoded string that does not
* begin with \\\?\\ nor \\\?\\UNC\\
*
* @see IsValidUTF16 to see whether the constructor
* succeeded
*/
Utf8ToFilename::Utf8ToFilename( const string &utf8string )
: _wideCharString( NULL )
, utf8( _utf8 )
{
// See
// http://msdn.microsoft.com/en-us/library/aa365247%28v=vs.85%29.aspx
// for notes about path lengths, prefixes, etc. The
// goal is to support the longest path possible.
// Relative paths are limited to 260 characters but
// absolute paths can be up to about 32767
// characters if properly prefixed.
// If utf8string is a relative path, convert it to
// UTF-16 and be done.
if (!IsAbsolute(utf8string))
{
_wideCharString = ConvertToUTF16(utf8string);
return;
}
// Since the prefix has backslashes, convert any forward
// slashes in utf8string to backslashes to keep Windows
// happy
const string *utf8ToUse = &utf8string;
string forwardSlash;
if (utf8string.find('/') != std::string::npos)
{
forwardSlash = utf8string;
std::replace(forwardSlash.begin(),forwardSlash.end(),'/','\\');
utf8ToUse = &forwardSlash;
}
ASSERT(utf8ToUse);
ASSERT((*utf8ToUse).length() > 0);
// utf8string is an absolute path. It could be a
// UNC path (\\host\path). The prefix is different
// for UNC paths than it is for non-UNC paths.
string prefixedPath;
if (IsUncPath(*utf8ToUse))
{
// utf8string begins with two backslashes, but
// with a prefix we only need one so we can't
// just prepend a prefix.
prefixedPath = "\\\\?\\UNC" + (*utf8ToUse).substr(1);
}
else
{
prefixedPath = "\\\\?\\" + *utf8ToUse;
}
// Transform prefixedPath to UTF-16 so it's
// appropriate for CreateFileW
_wideCharString = ConvertToUTF16(prefixedPath);
}
Utf8ToFilename::~Utf8ToFilename( )
{
if( _wideCharString != NULL )
{
free(_wideCharString);
_wideCharString = NULL;
}
}
/**
* Convert a UTF-8 encoded string to a UTF-16 string
*
* @param utf8 the NUL-terminated UTF-8 string to decode
*
* @retval NULL error allocating memory for UTF-16 string
*
* @retval non-NULL NUL-terminated UTF-16 version of @p
* utf8. Invalid portions of UTF-8 are represented by a
* replacement character U+FFFD. The caller is
* responsible for freeing this memory.
*/
wchar_t *
Utf8ToFilename::ConvertToUTF16 ( const string &utf8string )
{
int num_bytes;
size_t num_chars;
wchar_t *retval;
ASSERT(sizeof(wchar_t) == 2);
// Store the utf8 string in our member variable so it's
// available
_utf8 = utf8string;
// We need to find out how many characters we're dealing
// with so we know how much memory to allocate. At the
// same time, it's possible that the string we've been
// given isn't valid UTF-8. So, just use the length of
// the string we've been given as the number of
// characters to allocate. The decoded string can't be
// longer than this, even taking into account surrogate
// pairs since they require 4 UTF-8 characters but only
// two UTF-16 character elements.
num_chars = utf8string.length();
LOG_PRINTF((MP4_LOG_VERBOSE4,"%s: entry point (%d character string)",
__FUNCTION__,num_chars));
/*
** Allocate space for the decoded string. Add one
** for the NUL terminator.
*/
num_bytes = (num_chars + 1) * sizeof(wchar_t);
retval = (wchar_t *)malloc(num_bytes);
if (!retval)
{
log.errorf("%s: error allocating memory for %d byte(s)",__FUNCTION__,num_bytes);
return NULL;
}
/*
** ConvertToUTF16Buf zeroes out the memory so don't
** do it here
*/
// ConvertToUTF16Buf shouldn't fail if we allocated
// enough memory for the entire string. Check
// anyway just to be safe.
if (!ConvertToUTF16Buf(utf8string.c_str(),retval,num_bytes))
{
// But ASSERT so we can find the problem and fix
// it.
ASSERT(0);
free(retval);
retval = NULL;
return NULL;
}
return retval;
}
/**
* Convert a UTF-8 encoded string to a UTF-16 string in
* a previously allocated buffer.
*
* @param utf8 the NUL-terminated UTF-8 string to decode
*
* @param utf16_buf the buffer in which to place the
* UTF-16 version of @p utf8. If there's enough space
* to hold a NUL terminator, @p utf16_buf contains one.
* If not, @p utf16_buf is not NUL terminated.
*
* @param num_bytes the number of bytes that @p
* utf16_str points to
*
* @retval 0 error converting @p name to UTF-16,
* including when @p utf8 requires more space to encode
* in UTF-16 than indicated by @p num_bytes. In that
* case, @p utf16_buf contains the UTF-16 encoding of as
* much of @p utf8 as possible.
*
* @retval 1 successfully converted @p name to @p UTF-16
* in @p utf16_buf. wide character (UTF-16) version of
* @p Invalid portions of UTF-8 are represented by a
* replacement character U+FFFD.
*/
int
Utf8ToFilename::ConvertToUTF16Buf ( const char *utf8,
wchar_t *utf16_buf,
size_t num_bytes )
{
size_t i;
const UINT8 *next_char;
size_t num_chars;
size_t num_utf16_chars;
size_t num_input_bytes;
const UINT8 *p;
wchar_t this_utf16[2];
ASSERT(utf8);
ASSERT(utf16_buf || (num_bytes == 0));
ASSERT(sizeof(wchar_t) == 2);
ASSERT(num_bytes % sizeof(wchar_t) == 0);
LOG_PRINTF((MP4_LOG_VERBOSE4,"%s: converting \"%s\"",__FUNCTION__,utf8));
num_chars = strlen(utf8);
// If the input is NUL-terminated (which it better
// be), the NUL-terminator is a valid input byte as
// well
num_input_bytes = num_chars + 1;
// Make sure the buffer we've been given is long
// enough. We might need one UTF-16 character for
// every UTF-8 character. And one more for the NUL
// terminator.
//
// Here, check that there's room for a NUL
// terminator in the output string. This makes it
// safe to dereference p in the while loop below.
// It's probably enough to check num_bytes == 0 here
// but if we did that we'd have to change the error
// message after the while loop to be less specific.
// This way we give the caller more info about the
// input string.
if (num_bytes < sizeof(wchar_t))
{
log.errorf("%s: %u byte(s) is not enough to transform a %u byte UTF-8 string "
"to NUL-terminated UTF-16",__FUNCTION__,num_bytes,num_input_bytes);
return 0;
}
ASSERT(num_bytes > 0);
ASSERT(utf16_buf);
memset(utf16_buf,0,num_bytes);
// The number of UTF-16 characters we've got space for
// in utf16_buf
num_utf16_chars = num_bytes / sizeof(wchar_t);
p = (const UINT8 *)utf8;
i = 0;
while (*p && (i < num_utf16_chars))
{
LOG_PRINTF((MP4_LOG_VERBOSE4,"%s: decoding first UTF-8 byte 0x%02X (UTF-16 "
"character %d of at most %d)",__FUNCTION__,*p,(i + 1),
num_utf16_chars));
memset(this_utf16,0,sizeof(this_utf16));
// This function decodes illegal bytes/sequences
// with a replacement character and returns the
// pointer to the next character to decode. Pass
// NULL since we don't care about detecting invalid
// characters here.
next_char = Utf8DecodeChar(p,num_input_bytes,this_utf16,NULL);
// We've always got one character to assign
utf16_buf[i++] = this_utf16[0];
// If we're dealing with a surrogate pair,
// assign the low half too
if (this_utf16[1])
{
// We may not have any more room in the
// UTF-16 buffer. Check to make sure we
// don't step on someone else's memory. We
// need to return failure here instead of
// depending on our other logic to do it for
// us. We'll get out of the while loop with
// no extra code, but if we're dealing with
// the UTF-16 encoding of the last character
// in the input string, there won't appear
// to be anything wrong.
if (i >= num_utf16_chars)
{
log.errorf("%s: out of space in %u byte output string to store surrogate "
"pair low half (0x%04X)",__FUNCTION__,num_bytes,this_utf16[1]);
return 0;
}
utf16_buf[i++] = this_utf16[1];
}
// Put this here to make it brutally clear that
// the cast is safe
ASSERT(next_char >= p);
num_input_bytes -= (size_t)(next_char - p);
p = next_char;
}
if (*p)
{
// Since num_input_bytes includes 1 for the
// NUL-terminator, it's got to be bigger than
// one here.
ASSERT(num_input_bytes > 1);
log.errorf("%s: %u byte(s) of input string remain(s) undecoded (%s): out of space in "
"%u byte output string",__FUNCTION__,(num_input_bytes - 1),p,num_bytes);
return 0;
}
return 1;
}
/**
* Accessor for the length of a prefix (i.e. \\\?\\ or
* \\\?\\UNC\\) that begins a filename
*
* @param utf8string the UTF-8 encoded filename to
* examine
*
* @return the length of the prefix of @p utf8string in
* characters
*/
int
Utf8ToFilename::GetPrefixLen ( const string &utf8string )
{
if (utf8string.find("\\\\?\\") == 0)
{
return strlen("\\\\?\\");
}
if (utf8string.find("\\\\?\\UNC\\") == 0)
{
return strlen("\\\\?\\UNC\\");
}
return 0;
}
/**
* Determine if a path is absolute or not
*
* @param utf8string the UTF-8 encoded path to examine
* that does not begin with \\\?\\ nor \\\?\\UNC\\
*
* @retval 0 @p utf8string is not an absolute path
* @retval 1 @p utf8string is an absolute path
*/
int
Utf8ToFilename::IsAbsolute ( const string &utf8string )
{
// Assume utf8string doesn't already start with a
// long filename prefix (i.e. \\?\ or \\?\UNC\)
// since the logic here depends on that.
ASSERT(GetPrefixLen(utf8string) == 0);
// Is an empty string absolute or relative? It's
// not absolute since we can't tell what
// drive/volume it's for so say it's relative.
if (utf8string.length() == 0)
{
return 0;
}
// Here we're looking for:
// x: drive relative
// x:\ absolute path
if (utf8string[1] == ':')
{
// It starts with x:, but is it x:/ ?
if ((utf8string.length() >= 2) && IsPathSeparator(utf8string[2]))
{
// Yup -- it's absolute
return 1;
}
// Nope, not x:/, just x:something
return 0;
}
// UNC paths are absolute paths too
return IsUncPath(utf8string);
}
/**
* Determine if a character is a valid path separator
*
* @param c the character to check
*
* @retval 0 @p c is not a valid path separator
* @retval 1 @p c is a valid path separator
*/
int
Utf8ToFilename::IsPathSeparator ( char c )
{
return ((c == '\\') || (c == '/'));
}
/**
* Determine if a path is a UNC path
*
* @param utf8string the UTF-8 encoded path to examine
* that does not begin with \\\?\\ nor \\\?\\UNC\\
*
* @retval 0 @p utf8string is not a UNC path
* @retval 1 @p utf8string is a UNC path
*/
int
Utf8ToFilename::IsUncPath ( const string &utf8string )
{
const char *host;
int num_slashes;
const char *p;
// Assume utf8string doesn't already start with a
// long filename prefix (i.e. \\?\ or \\?\UNC\)
// since the logic here depends on that.
ASSERT(GetPrefixLen(utf8string) == 0);
// Is an empty string a UNC path? No.
if (utf8string.length() == 0)
{
return 0;
}
// Recognize:
// //volume/path
// \\volume\path
if (!IsPathSeparator(utf8string[0]))
{
// If it doesn't start with a path separator, it's
// not a UNC path.
return 0;
}
// The path starts with a slash, so it could be a UNC
// path. See if it starts with two slashes...Be careful
// though, it might have more than 2 slashes.
p = utf8string.c_str();
num_slashes = 0;
while (*p && IsPathSeparator(*p))
{
num_slashes++;
p++;
}
// We found a slash at the beginning so we better have
// at least one here
ASSERT(num_slashes >= 1);
if ((num_slashes > 2) || !(*p))
{
// If we've got more than two slashes or we've
// run off the end of the string (///foo or
// //)...who knows how the OS will handle it,
// but it's not a UNC path.
log.errorf("%s: don't understand path(%s)",__FUNCTION__,utf8string.c_str());
return 0;
}
// If we've only got one slash, it looks like a
// drive relative path. If it's something like
// /foo//bar it's not clear how the OS handles it,
// but that's someone else's problem. It's not a
// UNC path.
if (num_slashes == 1)
{
return 0;
}
// If we're here, we've got two slashes followed by
// a non-slash. Something like //foo. To be a
// proper UNC path, we need to see a hostname
// (e.g. foo), and then another slash. If not, it's
// not a UNC path.
ASSERT(num_slashes == 2);
// Tempting to use STRTOK_R here, but that modifies
// the original string. Instead of making a copy,
// search manually.
host = p;
while (*p && !IsPathSeparator(*p))
{
p++;
}
// We checked for separators above, so we better
// have moved on at least a bit
ASSERT(host != p);
if (!(*p))
{
// We ran off the end of the string without finding
// another separator. So, we've got something like
//
// //foobar
//
// which isn't a UNC path.
log.warningf("%s: incomplete UNC path: host only(%s)",__FUNCTION__,
utf8string.c_str());
return 0;
}
// p points to a separator, so...we've got one of:
// //host//
// //host//blah
// //host/bar
//
// Of these, only the last is a proper UNC path. See
// what we've got after p.
num_slashes = 0;
while (*p && IsPathSeparator(*p))
{
num_slashes++;
p++;
}
// We better have at least one slash or our logic is
// broken
ASSERT(num_slashes >= 1);
if (!(*p))
{
// //host// (or maybe //host///), but no path
// part after the host
log.warningf("%s: incomplete UNC path: no path after host(%s)",
__FUNCTION__,utf8string.c_str());
return 0;
}
if (num_slashes > 1)
{
// Another busted case //host//blah or
// //host///blah, etc.
log.warningf("%s: invalid UNC path: too many slashes after host(%s)",
__FUNCTION__,utf8string.c_str());
return 0;
}
// If we're here it means num_slashes is exactly 1
// so we've got //host/something so we're calling
// that a UNC path.
return 1;
}
/**
* Accessor for whether the UTF-16 encoded string is valid
*
* @retval false the UTF-16 encoded string is not valid
* @retval true the UTF-16 encoded string is valid
*/
bool
Utf8ToFilename::IsUTF16Valid( ) const
{
return (_wideCharString ? true : false);
}
/**
* Decode one UTF-8 encoded character into a UTF-16
* character. The trouble here is that UTF-16 is really a
* variable length encoding to handle surrogate pairs
* (0xD800 --> 0xDFFF). This way UTF-16 can handle more
* than 2^16 characters. So we need to be careful. UCS-2
* is a fixed width (16-bit) encoding that we could use, but
* then we can only handle 2^16 characters (the BMP). To
* handle all 2^21 characters, we need UTF-16.
*
* What does Windows really use? UTF-16. See
* http://unicode.org/iuc/iuc17/b2/slides.ppt for a
* discussion.
* http://discuss.fogcreek.com/joelonsoftware5/default.asp?cmd=show&ixPost=168543
* also has some info.
*
* @param utf8_char the UTF-8 character to decode, possibly
* occupying multiple bytes, not necessarily NUL terminated
*
* @param num_bytes the number of bytes that @p utf8_char
* points to (must be > 0)
*
* @param utf16 populated with the UTF-16 equivalent of @p
* utf8_char. Note that this must point to at least 2
* wchar_t's of memory so there's room to hold a surrogate
* pair.
*
* @param invalid populated with 1 if @p utf8_char doesn't
* point to a valid UTF-8 encoded character, 0 if @p
* utf8_char is valid.
*
* @return the next byte to examine for subsequent decoding
* (some number of bytes after @p utf8_char). This may not
* be valid to dereference depending on the value of @p
* num_bytes.
*/
const UINT8 *
Utf8ToFilename::Utf8DecodeChar ( const UINT8 *utf8_char,
size_t num_bytes,
wchar_t *utf16,
int *invalid )
{
wchar_t high_half;
int i;
UINT8 len;
wchar_t low_half;
UINT8 mask;
const UINT8 *p;
UINT32 ucs4;
int valid_len;
ASSERT(utf8_char);
ASSERT(num_bytes > 0);
ASSERT(utf16);
LOG_PRINTF((MP4_LOG_VERBOSE4,"%s: decoding UTF-8 string at address 0x%p",
__FUNCTION__,utf8_char));
/*
** Assume utf8_char is invalid until we learn otherwise
*/
if (invalid)
{
*invalid = 1;
}
/*
** Traverse the UTF-8 encoding and figure out what we've
** got.
*/
p = (const UINT8 *)(utf8_char);
/*
** This is the number of bytes we expect based on the
** first octet. If subsequent bytes are NUL or invalid,
** then it may not the same as the actual len.
*/
len = Utf8NumOctets(*p);
if (len == 0)
{
log.errorf("%s: 0x%02X is not a valid first byte of a UTF-8 encoded character",__FUNCTION__,*p);
/*
** Use the replacement character and advance past
** the invalid byte
*/
*utf16 = REPLACEMENT_CHAR;
return p + 1;
}
/*
** Handle one byte encodings in a special case. See
** below for an explanation of how we mask successive
** bytes of an encoding to see why. We're depending on
** the validation in Utf8NumOctets here to make this OK.
*/
if (len == 1)
{
/*
** There's no intermediate UCS-4 step here. We go
** straight to UTF-16 since they're the same.
*/
LOG_PRINTF((MP4_LOG_VERBOSE4,"%s: one byte UTF-16 encoding: 0x%02X",
__FUNCTION__,*p));
*utf16 = *p;
if (invalid)
{
*invalid = 0;
}
return p + 1;
}
/*
** Make sure we've got enough bytes in our input string
** to form a valid UTF-8 character
*/
if (len > num_bytes)
{
log.errorf("%s: first byte 0x%02X indicates a %d byte "
"UTF-8 character, but we only have %u valid byte(s)",
__FUNCTION__,*p,len,num_bytes);
*utf16 = REPLACEMENT_CHAR;
return p + 1;
}
/*
** Traverse the bytes that should be part of this UTF-8
** encoded character and make sure we don't have an
** overlength encoding, and make sure that each
** character is valid.
*/
/*
** As we traverse each character, we mask off the
** appropriate number of bits and include them in the
** overall result.
**
** 1 byte encoding [U+00000000,U+0000007F]: 7 bits (7 bits total) (handled above)
** 2 byte encoding [U+00000080,U+000007FF]: 5 bits, 6 bits (11 bits total)
** 3 byte encoding [U+00000800,U+0000FFFF]: 4 bits, 6 bits, 6 bits (16 bits total)
** 4 byte encoding [U+00010000,U+001FFFFF]: 3 bits, 6 bits, 6 bits, 6 bits (21 bits total)
** 5 byte encoding [U+00200000,U+03FFFFFF]: 2 bits, 6 bits, 6 bits, 6 bits, 6 bits (26 bits total)
** 6 byte encoding [U+04000000,U+7FFFFFFF]: 1 bit, 6 bits, 6 bits, 6 bits, 6 bits, 6 bits (31 bits total)
**
** So, mask the initial byte appropriately, then take
** the bottom 6 bits from the remaining bytes. To be
** brutally explicit, the first byte mask is:
**
** 1 byte encoding: 0x7F (or 0x80 - 1) (or (1 << 7) - 1)
** 2 byte encoding: 0x1F (or 0x20 - 1) (or (1 << 5) - 1)
** 3 byte encoding: 0x0F (or 0x10 - 1) (or (1 << 4) - 1)
** 4 byte encoding: 0x07 (or 0x08 - 1) (or (1 << 3) - 1)
** 5 byte encoding: 0x03 (or 0x04 - 1) (or (1 << 2) - 1)
** 6 byte encoding: 0x01 (or 0x02 - 1) (or (1 << 1) - 1)
**
** So, the one byte encoding is a special case (again,
** handled above), but for the other lengths, the mask
** is (1 << (7 - len)) - 1.
*/
/*
** Handle the first byte of multi-byte encodings since
** it's special
*/
ASSERT(len > 1);
ASSERT(len <= 6);
mask = (1 << (7 - len)) - 1;
ucs4 = *p & mask;
p++;
/*
** Now handle the remaining bytes
*/
for (i = 1;(i < len);i++)
{
if ((*p < 0x80) || (*p > 0xBF))
{
log.errorf("%s: 0x%02X is not a valid continuation character in a UTF-8 encoding",
__FUNCTION__,*p);
/*
** Use the replacement character and return the
** next byte after the invalid sequence as the
** place for subsequent decoding operations. In
** this case the invalid continuation character
** could be the beginning of the next valid
** sequence, so return that.
*/
*utf16 = REPLACEMENT_CHAR;
return p;
}
/*
** For the remainder of the bytes, shift over what
** we've already got by 6 bits, and then OR in the
** bottom 6 bits of the current byte.
*/
ucs4 = (ucs4 << 6) | (*p & 0x3F);
p++;
}
/*
** p is now pointing to the beginning of the next UTF-8
** sequence to decode...
*/
/*
** Finally, detect overlong encodings. For example, a
** line feed (U+000A) should be encoded as 0x0A
** (0b00001010) but could in theory be encoded in UTF-8
** as 0xC0 0x8A (0b10001010).
**
** Another example is the forward slash (/) (U+002F).
** It should be encoded as 0x2F, but could in theory be
** encoded in UTF-8 as 0xC0 0xAF (which we'll catch
** because 0xC0 is an invalid first byte of a UTF-8
** encoding), but could also be 0xE0 0x80 0xAF.
**
** I can't see any reasonable way to do this other than
** to check the decoded character against its expected
** length
*/
valid_len = Utf8LenFromUcs4(ucs4);
if (valid_len == 0)
{
/*
** This should never happen
*/
log.errorf("%s: decoded a character that we can't encode again (0x%08X)",__FUNCTION__,ucs4);
ASSERT(0);
/*
** If it does, use the replacement character
*/
*utf16 = REPLACEMENT_CHAR;
return p;
}
if (len != valid_len)
{
ASSERT(len > valid_len);
log.errorf("%s: overlong encoding(%s)...should be %d byte(s), not %d",__FUNCTION__,
utf8_char,valid_len,len);
*utf16 = REPLACEMENT_CHAR;
return p;
}
/*
** UTF-16 can only hold 21 bits. As of now (21-dec-10),
** there's no Unicode code point bigger than 2^21. To
** be safe, check...
*/
if (ucs4 > 0x0010FFFF)
{
log.errorf("%s: code point 0x%08X is too big",__FUNCTION__,ucs4);
*utf16 = REPLACEMENT_CHAR;
return p;
}
/*
** Check to make sure we're not working with a "code
** point" that is in the range used to indicate
** surrogate pairs.
*/
if ((ucs4 >= 0x0000D800) && (ucs4 <= 0x0000DFFF))
{
log.errorf("%s: code point 0x%08X is in the range used to indicate surrogate pairs",
__FUNCTION__,ucs4);
*utf16 = REPLACEMENT_CHAR;
return p;
}
/*
** To (try to) be complete, check for a couple more
** invalid code points
*/
if ((ucs4 == 0x0000FFFF) || (ucs4 == 0x0000FFFE))
{
log.errorf("%s: invalid code point (0x%08X)",__FUNCTION__,ucs4);
*utf16 = REPLACEMENT_CHAR;
return p;
}
/*
** Finally, convert from UCS-4 to UTF-16. This may be a
** straightforward assignment, but we have to deal with
** surrogate pairs
*/
if (ucs4 <= 0x0000FFFF)
{
*utf16 = ucs4 & 0xFFFF;
LOG_PRINTF((MP4_LOG_VERBOSE4,"%s: UTF-16 encoding of 0x%08X is 0x%04X",
__FUNCTION__,ucs4,*utf16));
if (invalid)
{
*invalid = 0;
}
return p;
}
/*
** Transform UCS-4 into a UTF-16 surrogate pair
*/
/*
** Grab bits [10,20] (where bit 0 is the LSB) and shift
** them down
*/
high_half = 0xD800 + ((ucs4 - 0x00010000) >> 10);
/*
** And the bottom 10 bits [0,9]
*/
low_half = 0xDC00 + (ucs4 & 0x03FF);
utf16[0] = high_half;
utf16[1] = low_half;
LOG_PRINTF((MP4_LOG_VERBOSE4,"%s: UTF-16 encoding of 0x%08X is 0x%04X:0x%04X",
__FUNCTION__,ucs4,utf16[0],utf16[1]));
if (invalid)
{
*invalid = 0;
}
return p;
}
/**
* Determine the number of bytes required to hold the UTF-8
* encoding of a UCS-4 code point
*
* @param ucs4 the code point
*
* @param use_syslog 1 to use syslog, 0 otherwise
*
* @retval 0 @p ucs4 is not a valid code point
*
* @retval [1,6] the number of bytes required to hold the
* UTF-8 encoding of @p ucs4
*/
size_t
Utf8ToFilename::Utf8LenFromUcs4 ( UINT32 ucs4 )
{
size_t table_idx;
LOG_PRINTF((MP4_LOG_VERBOSE4,"%s: processing UCS-4 code point 0x%08X",
__FUNCTION__,ucs4));
for (table_idx = 0;(table_idx < (sizeof(s_len_info) /
sizeof(struct utf8_len_info)));
table_idx++)
{
if ((s_len_info[table_idx].range_min <= ucs4) &&
(ucs4 <= s_len_info[table_idx].range_max))
{
return s_len_info[table_idx].num_chars;
}
}
log.errorf("%s: 0x%08X is an invalid code point",__FUNCTION__,ucs4);
return 0;
}
/**
* Determine the number of octets that a UTF-8 encoded
* character should occupy based on its first byte
*
* @param utf8_first_byte the byte to examine
*
* @retval 0 @p utf8_first_byte is not a valid first byte of
* a UTF-8 encoded character
*
* @retval [1,6] the number of octets that @p
* utf8_first_byte should occupy
*/
UINT8
Utf8ToFilename::Utf8NumOctets ( UINT8 utf8_first_byte )
{
/**
* Here's a mapping from the first byte of a UTF-8
* character to the number of bytes it should contain
* based on information from
* http://www.unicode.org/versions/corrigendum1.html as
* well as
* http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
*
* [0x00,0x7F]: 1 (0-127) (128 possible values)
* [0x80,0xBF]: invalid (128-191) (64 possible values)
* [0xC0,0xDF]: 2 (192-223) (32 possible values) (see below)
* [0xE0,0xEF]: 3 (224-239) (16 possible values)
* [0xF0,0xF7]: 4 (240 - 247) (8 possible values)
* [0xF8,0xFB]: 5 (248 - 251) (4 possible values)
* [0xFC,0xFD]: 6 (252 - 253) (2 possible values)
* [0xFE,0xFF]: invalid (254 - 255) (2 possible values)
*
* There's some gray area about 0xC0 and 0xC1. It's
* clear they are invalid first bytes but the question
* is how to handle it. If I reject them here, they'll
* get replaced with the REPLACEMENT character. But, if
* I allow them here, it's likely that both this byte
* and the subsequent one will get replaced with only
* one replacement character. This is what
* http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
* assumes in sections 4.1.1, 4.2.1 and 4.3.1.
*/
if (utf8_first_byte <= 0x7F)
{
return 1;
}
if ((utf8_first_byte >= 0x80) && (utf8_first_byte <= 0xBF))
{
return 0;
}
if ((utf8_first_byte >= 0xC0) && (utf8_first_byte <= 0xDF))
{
return 2;
}
if ((utf8_first_byte >= 0xE0) && (utf8_first_byte <= 0xEF))
{
return 3;
}
if ((utf8_first_byte >= 0xF0) && (utf8_first_byte <= 0xF7))
{
return 4;
}
if ((utf8_first_byte >= 0xF8) && (utf8_first_byte <= 0xFB))
{
return 5;
}
if ((utf8_first_byte >= 0xFC) && (utf8_first_byte <= 0xFD))
{
return 6;
}
ASSERT((utf8_first_byte == 0xFE) || (utf8_first_byte == 0xFF));
return 0;
}
///////////////////////////////////////////////////////////////////////////////
}}} // namespace mp4v2::platform::win32