/****************************************************************************
**
** Copyright (C) 2019 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the utils of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:GPL-EXCEPT$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include <qlist.h>
#include <qhash.h>
#include <qfile.h>
#include <qbytearray.h>
#include <qstring.h>
#include <qchar.h>
#include <qvector.h>
#include <qdebug.h>
#if 0
#include <private/qunicodetables_p.h>
#endif

#define DATA_VERSION_S "13.0"
#define DATA_VERSION_STR "QChar::Unicode_13_0"


static QHash<QByteArray, QChar::UnicodeVersion> age_map;

static void initAgeMap()
{
    struct AgeMap {
        const QChar::UnicodeVersion version;
        const char *age;
    } ageMap[] = {
        { QChar::Unicode_1_1,   "1.1" },
        { QChar::Unicode_2_0,   "2.0" },
        { QChar::Unicode_2_1_2, "2.1" },
        { QChar::Unicode_3_0,   "3.0" },
        { QChar::Unicode_3_1,   "3.1" },
        { QChar::Unicode_3_2,   "3.2" },
        { QChar::Unicode_4_0,   "4.0" },
        { QChar::Unicode_4_1,   "4.1" },
        { QChar::Unicode_5_0,   "5.0" },
        { QChar::Unicode_5_1,   "5.1" },
        { QChar::Unicode_5_2,   "5.2" },
        { QChar::Unicode_6_0,   "6.0" },
        { QChar::Unicode_6_1,   "6.1" },
        { QChar::Unicode_6_2,   "6.2" },
        { QChar::Unicode_6_3,   "6.3" },
        { QChar::Unicode_7_0,   "7.0" },
        { QChar::Unicode_8_0,   "8.0" },
        { QChar::Unicode_9_0,   "9.0" },
        { QChar::Unicode_10_0,   "10.0" },
        { QChar::Unicode_11_0,   "11.0" },
        { QChar::Unicode_12_0,   "12.0" },
        { QChar::Unicode_12_1,   "12.1" }, // UCD Revision 24
        { QChar::Unicode_13_0,   "13.0" }, // UCD Revision 26
        { QChar::Unicode_Unassigned, 0 }
    };
    AgeMap *d = ageMap;
    while (d->age) {
        age_map.insert(d->age, d->version);
        ++d;
    }
}

static QHash<QByteArray, QChar::Category> categoryMap;

static void initCategoryMap()
{
    struct Cat {
        QChar::Category cat;
        const char *name;
    } categories[] = {
        { QChar::Mark_NonSpacing,          "Mn" },
        { QChar::Mark_SpacingCombining,    "Mc" },
        { QChar::Mark_Enclosing,           "Me" },

        { QChar::Number_DecimalDigit,      "Nd" },
        { QChar::Number_Letter,            "Nl" },
        { QChar::Number_Other,             "No" },

        { QChar::Separator_Space,          "Zs" },
        { QChar::Separator_Line,           "Zl" },
        { QChar::Separator_Paragraph,      "Zp" },

        { QChar::Other_Control,            "Cc" },
        { QChar::Other_Format,             "Cf" },
        { QChar::Other_Surrogate,          "Cs" },
        { QChar::Other_PrivateUse,         "Co" },
        { QChar::Other_NotAssigned,        "Cn" },

        { QChar::Letter_Uppercase,         "Lu" },
        { QChar::Letter_Lowercase,         "Ll" },
        { QChar::Letter_Titlecase,         "Lt" },
        { QChar::Letter_Modifier,          "Lm" },
        { QChar::Letter_Other,             "Lo" },

        { QChar::Punctuation_Connector,    "Pc" },
        { QChar::Punctuation_Dash,         "Pd" },
        { QChar::Punctuation_Open,         "Ps" },
        { QChar::Punctuation_Close,        "Pe" },
        { QChar::Punctuation_InitialQuote, "Pi" },
        { QChar::Punctuation_FinalQuote,   "Pf" },
        { QChar::Punctuation_Other,        "Po" },

        { QChar::Symbol_Math,              "Sm" },
        { QChar::Symbol_Currency,          "Sc" },
        { QChar::Symbol_Modifier,          "Sk" },
        { QChar::Symbol_Other,             "So" },
        { QChar::Other_NotAssigned, 0 }
    };
    Cat *c = categories;
    while (c->name) {
        categoryMap.insert(c->name, c->cat);
        ++c;
    }
}


static QHash<QByteArray, QChar::Decomposition> decompositionMap;

static void initDecompositionMap()
{
    struct Dec {
        QChar::Decomposition dec;
        const char *name;
    } decompositions[] = {
        { QChar::Canonical, "<canonical>" },
        { QChar::Font, "<font>" },
        { QChar::NoBreak, "<noBreak>" },
        { QChar::Initial, "<initial>" },
        { QChar::Medial, "<medial>" },
        { QChar::Final, "<final>" },
        { QChar::Isolated, "<isolated>" },
        { QChar::Circle, "<circle>" },
        { QChar::Super, "<super>" },
        { QChar::Sub, "<sub>" },
        { QChar::Vertical, "<vertical>" },
        { QChar::Wide, "<wide>" },
        { QChar::Narrow, "<narrow>" },
        { QChar::Small, "<small>" },
        { QChar::Square, "<square>" },
        { QChar::Compat, "<compat>" },
        { QChar::Fraction, "<fraction>" },
        { QChar::NoDecomposition, 0 }
    };
    Dec *d = decompositions;
    while (d->name) {
        decompositionMap.insert(d->name, d->dec);
        ++d;
    }
}


enum Direction {
    DirL = QChar::DirL,
    DirR = QChar::DirR,
    DirEN = QChar::DirEN,
    DirES = QChar::DirES,
    DirET = QChar::DirET,
    DirAN = QChar::DirAN,
    DirCS = QChar::DirCS,
    DirB = QChar::DirB,
    DirS = QChar::DirS,
    DirWS = QChar::DirWS,
    DirON = QChar::DirON,
    DirLRE = QChar::DirLRE,
    DirLRO = QChar::DirLRO,
    DirAL = QChar::DirAL,
    DirRLE = QChar::DirRLE,
    DirRLO = QChar::DirRLO,
    DirPDF = QChar::DirPDF,
    DirNSM = QChar::DirNSM,
    DirBN = QChar::DirBN,
    DirLRI = QChar::DirLRI,
    DirRLI = QChar::DirRLI,
    DirFSI = QChar::DirFSI,
    DirPDI = QChar::DirPDI,

    Dir_Unassigned
};

static QHash<QByteArray, Direction> directionMap;

static void initDirectionMap()
{
    struct Dir {
        Direction dir;
        const char *name;
    } directions[] = {
        { DirL, "L" },
        { DirR, "R" },
        { DirEN, "EN" },
        { DirES, "ES" },
        { DirET, "ET" },
        { DirAN, "AN" },
        { DirCS, "CS" },
        { DirB, "B" },
        { DirS, "S" },
        { DirWS, "WS" },
        { DirON, "ON" },
        { DirLRE, "LRE" },
        { DirLRO, "LRO" },
        { DirAL, "AL" },
        { DirRLE, "RLE" },
        { DirRLO, "RLO" },
        { DirPDF, "PDF" },
        { DirNSM, "NSM" },
        { DirBN, "BN" },
        { DirLRI, "LRI" },
        { DirRLI, "RLI" },
        { DirFSI, "FSI" },
        { DirPDI, "PDI" },
        { Dir_Unassigned, 0 }
    };
    Dir *d = directions;
    while (d->name) {
        directionMap.insert(d->name, d->dir);
        ++d;
    }
}


enum JoiningType {
    Joining_None,
    Joining_Causing,
    Joining_Dual,
    Joining_Right,
    Joining_Left,
    Joining_Transparent,

    Joining_Unassigned
};

static QHash<QByteArray, JoiningType> joining_map;

static void initJoiningMap()
{
    struct JoiningList {
        JoiningType joining;
        const char *name;
    } joinings[] = {
        { Joining_None,        "U" },
        { Joining_Causing,     "C" },
        { Joining_Dual,        "D" },
        { Joining_Right,       "R" },
        { Joining_Left,        "L" },
        { Joining_Transparent, "T" },
        { Joining_Unassigned, 0 }
    };
    JoiningList *d = joinings;
    while (d->name) {
        joining_map.insert(d->name, d->joining);
        ++d;
    }
}


static const char *grapheme_break_class_string =
    "enum GraphemeBreakClass {\n"
    "    GraphemeBreak_Any,\n"
    "    GraphemeBreak_CR,\n"
    "    GraphemeBreak_LF,\n"
    "    GraphemeBreak_Control,\n"
    "    GraphemeBreak_Extend,\n"
    "    GraphemeBreak_ZWJ,\n"
    "    GraphemeBreak_RegionalIndicator,\n"
    "    GraphemeBreak_Prepend,\n"
    "    GraphemeBreak_SpacingMark,\n"
    "    GraphemeBreak_L,\n"
    "    GraphemeBreak_V,\n"
    "    GraphemeBreak_T,\n"
    "    GraphemeBreak_LV,\n"
    "    GraphemeBreak_LVT,\n"
    "    Graphemebreak_E_Base,\n"
    "    Graphemebreak_E_Modifier,\n"
    "    Graphemebreak_Glue_After_Zwj,\n"
    "    Graphemebreak_E_Base_GAZ,\n"
    "\n"
    "    NumGraphemeBreakClasses\n"
    "};\n\n";

enum GraphemeBreakClass {
    GraphemeBreak_Any,
    GraphemeBreak_CR,
    GraphemeBreak_LF,
    GraphemeBreak_Control,
    GraphemeBreak_Extend,
    GraphemeBreak_ZWJ,
    GraphemeBreak_RegionalIndicator,
    GraphemeBreak_Prepend,
    GraphemeBreak_SpacingMark,
    GraphemeBreak_L,
    GraphemeBreak_V,
    GraphemeBreak_T,
    GraphemeBreak_LV,
    GraphemeBreak_LVT,
    Graphemebreak_E_Base,
    Graphemebreak_E_Modifier,
    Graphemebreak_Glue_After_Zwj,
    Graphemebreak_E_Base_GAZ,

    GraphemeBreak_Unassigned
};

static QHash<QByteArray, GraphemeBreakClass> grapheme_break_map;

static void initGraphemeBreak()
{
    struct GraphemeBreakList {
        GraphemeBreakClass brk;
        const char *name;
    } breaks[] = {
        { GraphemeBreak_Any, "Any" },
        { GraphemeBreak_CR, "CR" },
        { GraphemeBreak_LF, "LF" },
        { GraphemeBreak_Control, "Control" },
        { GraphemeBreak_Extend, "Extend" },
        { GraphemeBreak_ZWJ, "ZWJ" },
        { GraphemeBreak_RegionalIndicator, "Regional_Indicator" },
        { GraphemeBreak_Prepend, "Prepend" },
        { GraphemeBreak_SpacingMark, "SpacingMark" },
        { GraphemeBreak_L, "L" },
        { GraphemeBreak_V, "V" },
        { GraphemeBreak_T, "T" },
        { GraphemeBreak_LV, "LV" },
        { GraphemeBreak_LVT, "LVT" },
        { Graphemebreak_E_Base, "E_Base" },
        { Graphemebreak_E_Modifier, "E_Modifier" },
        { Graphemebreak_Glue_After_Zwj, "Glue_After_Zwj" },
        { Graphemebreak_E_Base_GAZ, "E_Base_GAZ" },
        { GraphemeBreak_Unassigned, 0 }
    };
    GraphemeBreakList *d = breaks;
    while (d->name) {
        grapheme_break_map.insert(d->name, d->brk);
        ++d;
    }
}


static const char *word_break_class_string =
    "enum WordBreakClass {\n"
    "    WordBreak_Any,\n"
    "    WordBreak_CR,\n"
    "    WordBreak_LF,\n"
    "    WordBreak_Newline,\n"
    "    WordBreak_Extend,\n"
    "    WordBreak_ZWJ,\n"
    "    WordBreak_Format,\n"
    "    WordBreak_RegionalIndicator,\n"
    "    WordBreak_Katakana,\n"
    "    WordBreak_HebrewLetter,\n"
    "    WordBreak_ALetter,\n"
    "    WordBreak_SingleQuote,\n"
    "    WordBreak_DoubleQuote,\n"
    "    WordBreak_MidNumLet,\n"
    "    WordBreak_MidLetter,\n"
    "    WordBreak_MidNum,\n"
    "    WordBreak_Numeric,\n"
    "    WordBreak_ExtendNumLet,\n"
    "    WordBreak_E_Base,\n"
    "    WordBreak_E_Modifier,\n"
    "    WordBreak_Glue_After_Zwj,\n"
    "    WordBreak_E_Base_GAZ,\n"
    "    WordBreak_WSegSpace,\n"
    "\n"
    "    NumWordBreakClasses\n"
    "};\n\n";

enum WordBreakClass {
    WordBreak_Any,
    WordBreak_CR,
    WordBreak_LF,
    WordBreak_Newline,
    WordBreak_Extend,
    WordBreak_ZWJ,
    WordBreak_Format,
    WordBreak_RegionalIndicator,
    WordBreak_Katakana,
    WordBreak_HebrewLetter,
    WordBreak_ALetter,
    WordBreak_SingleQuote,
    WordBreak_DoubleQuote,
    WordBreak_MidNumLet,
    WordBreak_MidLetter,
    WordBreak_MidNum,
    WordBreak_Numeric,
    WordBreak_ExtendNumLet,
    WordBreak_E_Base,
    WordBreak_E_Modifier,
    WordBreak_Glue_After_Zwj,
    WordBreak_E_Base_GAZ,
    WordBreak_WSegSpace,

    WordBreak_Unassigned
};

static QHash<QByteArray, WordBreakClass> word_break_map;

static void initWordBreak()
{
    struct WordBreakList {
        WordBreakClass brk;
        const char *name;
    } breaks[] = {
        { WordBreak_Any, "Any" },
        { WordBreak_CR, "CR" },
        { WordBreak_LF, "LF" },
        { WordBreak_Newline, "Newline" },
        { WordBreak_Extend, "Extend" },
        { WordBreak_ZWJ, "ZWJ" },
        { WordBreak_Format, "Format" },
        { WordBreak_RegionalIndicator, "Regional_Indicator" },
        { WordBreak_Katakana, "Katakana" },
        { WordBreak_HebrewLetter, "Hebrew_Letter" },
        { WordBreak_ALetter, "ALetter" },
        { WordBreak_SingleQuote, "Single_Quote" },
        { WordBreak_DoubleQuote, "Double_Quote" },
        { WordBreak_MidNumLet, "MidNumLet" },
        { WordBreak_MidLetter, "MidLetter" },
        { WordBreak_MidNum, "MidNum" },
        { WordBreak_Numeric, "Numeric" },
        { WordBreak_ExtendNumLet, "ExtendNumLet" },
        { WordBreak_E_Base, "E_Base" },
        { WordBreak_E_Modifier, "E_Modifier" },
        { WordBreak_Glue_After_Zwj, "Glue_After_Zwj" },
        { WordBreak_E_Base_GAZ, "E_Base_GAZ" },
        { WordBreak_WSegSpace, "WSegSpace" },
        { WordBreak_Unassigned, 0 }
    };
    WordBreakList *d = breaks;
    while (d->name) {
        word_break_map.insert(d->name, d->brk);
        ++d;
    }
}


static const char *sentence_break_class_string =
    "enum SentenceBreakClass {\n"
    "    SentenceBreak_Any,\n"
    "    SentenceBreak_CR,\n"
    "    SentenceBreak_LF,\n"
    "    SentenceBreak_Sep,\n"
    "    SentenceBreak_Extend,\n"
    "    SentenceBreak_Sp,\n"
    "    SentenceBreak_Lower,\n"
    "    SentenceBreak_Upper,\n"
    "    SentenceBreak_OLetter,\n"
    "    SentenceBreak_Numeric,\n"
    "    SentenceBreak_ATerm,\n"
    "    SentenceBreak_SContinue,\n"
    "    SentenceBreak_STerm,\n"
    "    SentenceBreak_Close,\n"
    "\n"
    "    NumSentenceBreakClasses\n"
    "};\n\n";

enum SentenceBreakClass {
    SentenceBreak_Any,
    SentenceBreak_CR,
    SentenceBreak_LF,
    SentenceBreak_Sep,
    SentenceBreak_Extend,
    SentenceBreak_Sp,
    SentenceBreak_Lower,
    SentenceBreak_Upper,
    SentenceBreak_OLetter,
    SentenceBreak_Numeric,
    SentenceBreak_ATerm,
    SentenceBreak_SContinue,
    SentenceBreak_STerm,
    SentenceBreak_Close,

    SentenceBreak_Unassigned
};

static QHash<QByteArray, SentenceBreakClass> sentence_break_map;

static void initSentenceBreak()
{
    struct SentenceBreakList {
        SentenceBreakClass brk;
        const char *name;
    } breaks[] = {
        { SentenceBreak_Any, "Any" },
        { SentenceBreak_CR, "CR" },
        { SentenceBreak_LF, "LF" },
        { SentenceBreak_Sep, "Sep" },
        { SentenceBreak_Extend, "Extend" },
        { SentenceBreak_Extend, "Format" },
        { SentenceBreak_Sp, "Sp" },
        { SentenceBreak_Lower, "Lower" },
        { SentenceBreak_Upper, "Upper" },
        { SentenceBreak_OLetter, "OLetter" },
        { SentenceBreak_Numeric, "Numeric" },
        { SentenceBreak_ATerm, "ATerm" },
        { SentenceBreak_SContinue, "SContinue" },
        { SentenceBreak_STerm, "STerm" },
        { SentenceBreak_Close, "Close" },
        { SentenceBreak_Unassigned, 0 }
    };
    SentenceBreakList *d = breaks;
    while (d->name) {
        sentence_break_map.insert(d->name, d->brk);
        ++d;
    }
}


static const char *line_break_class_string =
    "// see http://www.unicode.org/reports/tr14/tr14-30.html\n"
    "// we don't use the XX and AI classes and map them to AL instead.\n"
    "enum LineBreakClass {\n"
    "    LineBreak_OP, LineBreak_CL, LineBreak_CP, LineBreak_QU, LineBreak_GL,\n"
    "    LineBreak_NS, LineBreak_EX, LineBreak_SY, LineBreak_IS, LineBreak_PR,\n"
    "    LineBreak_PO, LineBreak_NU, LineBreak_AL, LineBreak_HL, LineBreak_ID,\n"
    "    LineBreak_IN, LineBreak_HY, LineBreak_BA, LineBreak_BB, LineBreak_B2,\n"
    "    LineBreak_ZW, LineBreak_CM, LineBreak_WJ, LineBreak_H2, LineBreak_H3,\n"
    "    LineBreak_JL, LineBreak_JV, LineBreak_JT, LineBreak_RI, LineBreak_CB,\n"
    "    LineBreak_EB, LineBreak_EM, LineBreak_ZWJ,\n"
    "    LineBreak_SA, LineBreak_SG, LineBreak_SP,\n"
    "    LineBreak_CR, LineBreak_LF, LineBreak_BK,\n"
    "\n"
    "    NumLineBreakClasses\n"
    "};\n\n";

enum LineBreakClass {
    LineBreak_OP, LineBreak_CL, LineBreak_CP, LineBreak_QU, LineBreak_GL,
    LineBreak_NS, LineBreak_EX, LineBreak_SY, LineBreak_IS, LineBreak_PR,
    LineBreak_PO, LineBreak_NU, LineBreak_AL, LineBreak_HL, LineBreak_ID,
    LineBreak_IN, LineBreak_HY, LineBreak_BA, LineBreak_BB, LineBreak_B2,
    LineBreak_ZW, LineBreak_CM, LineBreak_WJ, LineBreak_H2, LineBreak_H3,
    LineBreak_JL, LineBreak_JV, LineBreak_JT, LineBreak_RI, LineBreak_CB,
    LineBreak_EB, LineBreak_EM, LineBreak_ZWJ,
    LineBreak_SA, LineBreak_SG, LineBreak_SP,
    LineBreak_CR, LineBreak_LF, LineBreak_BK,

    LineBreak_Unassigned
};

static QHash<QByteArray, LineBreakClass> line_break_map;

static void initLineBreak()
{
    // ### Classes XX and AI are left out and mapped to AL for now.
    // ### Class NL is mapped to BK.
    // ### Treating characters of class CJ as class NS will give CSS strict line breaking;
    //     treating them as class ID will give CSS normal breaking.
    struct LineBreakList {
        LineBreakClass brk;
        const char *name;
    } breaks[] = {
        { LineBreak_BK, "BK" },
        { LineBreak_CR, "CR" },
        { LineBreak_LF, "LF" },
        { LineBreak_CM, "CM" },
        { LineBreak_BK, "NL" },
        { LineBreak_SG, "SG" },
        { LineBreak_WJ, "WJ" },
        { LineBreak_ZW, "ZW" },
        { LineBreak_GL, "GL" },
        { LineBreak_SP, "SP" },
        { LineBreak_B2, "B2" },
        { LineBreak_BA, "BA" },
        { LineBreak_BB, "BB" },
        { LineBreak_HY, "HY" },
        { LineBreak_CB, "CB" },
        { LineBreak_NS, "CJ" },
        { LineBreak_CL, "CL" },
        { LineBreak_CP, "CP" },
        { LineBreak_EX, "EX" },
        { LineBreak_IN, "IN" },
        { LineBreak_NS, "NS" },
        { LineBreak_OP, "OP" },
        { LineBreak_QU, "QU" },
        { LineBreak_IS, "IS" },
        { LineBreak_NU, "NU" },
        { LineBreak_PO, "PO" },
        { LineBreak_PR, "PR" },
        { LineBreak_SY, "SY" },
        { LineBreak_AL, "AI" },
        { LineBreak_AL, "AL" },
        { LineBreak_HL, "HL" },
        { LineBreak_H2, "H2" },
        { LineBreak_H3, "H3" },
        { LineBreak_ID, "ID" },
        { LineBreak_JL, "JL" },
        { LineBreak_JV, "JV" },
        { LineBreak_JT, "JT" },
        { LineBreak_RI, "RI" },
        { LineBreak_SA, "SA" },
        { LineBreak_AL, "XX" },
        { LineBreak_EB, "EB" },
        { LineBreak_EM, "EM" },
        { LineBreak_ZWJ, "ZWJ" },
        { LineBreak_Unassigned, 0 }
    };
    LineBreakList *d = breaks;
    while (d->name) {
        line_break_map.insert(d->name, d->brk);
        ++d;
    }
}


static QHash<QByteArray, QChar::Script> scriptMap;

static void initScriptMap()
{
    struct Scrpt {
        QChar::Script script;
        const char *name;
    } scripts[] = {
        // general
        { QChar::Script_Unknown,                "Unknown" },
        { QChar::Script_Inherited,              "Inherited" },
        { QChar::Script_Common,                 "Common" },
        // pre-4.0
        { QChar::Script_Latin,                  "Latin" },
        { QChar::Script_Greek,                  "Greek" },
        { QChar::Script_Cyrillic,               "Cyrillic" },
        { QChar::Script_Armenian,               "Armenian" },
        { QChar::Script_Hebrew,                 "Hebrew" },
        { QChar::Script_Arabic,                 "Arabic" },
        { QChar::Script_Syriac,                 "Syriac" },
        { QChar::Script_Thaana,                 "Thaana" },
        { QChar::Script_Devanagari,             "Devanagari" },
        { QChar::Script_Bengali,                "Bengali" },
        { QChar::Script_Gurmukhi,               "Gurmukhi" },
        { QChar::Script_Gujarati,               "Gujarati" },
        { QChar::Script_Oriya,                  "Oriya" },
        { QChar::Script_Tamil,                  "Tamil" },
        { QChar::Script_Telugu,                 "Telugu" },
        { QChar::Script_Kannada,                "Kannada" },
        { QChar::Script_Malayalam,              "Malayalam" },
        { QChar::Script_Sinhala,                "Sinhala" },
        { QChar::Script_Thai,                   "Thai" },
        { QChar::Script_Lao,                    "Lao" },
        { QChar::Script_Tibetan,                "Tibetan" },
        { QChar::Script_Myanmar,                "Myanmar" },
        { QChar::Script_Georgian,               "Georgian" },
        { QChar::Script_Hangul,                 "Hangul" },
        { QChar::Script_Ethiopic,               "Ethiopic" },
        { QChar::Script_Cherokee,               "Cherokee" },
        { QChar::Script_CanadianAboriginal,     "CanadianAboriginal" },
        { QChar::Script_Ogham,                  "Ogham" },
        { QChar::Script_Runic,                  "Runic" },
        { QChar::Script_Khmer,                  "Khmer" },
        { QChar::Script_Mongolian,              "Mongolian" },
        { QChar::Script_Hiragana,               "Hiragana" },
        { QChar::Script_Katakana,               "Katakana" },
        { QChar::Script_Bopomofo,               "Bopomofo" },
        { QChar::Script_Han,                    "Han" },
        { QChar::Script_Yi,                     "Yi" },
        { QChar::Script_OldItalic,              "OldItalic" },
        { QChar::Script_Gothic,                 "Gothic" },
        { QChar::Script_Deseret,                "Deseret" },
        { QChar::Script_Tagalog,                "Tagalog" },
        { QChar::Script_Hanunoo,                "Hanunoo" },
        { QChar::Script_Buhid,                  "Buhid" },
        { QChar::Script_Tagbanwa,               "Tagbanwa" },
        { QChar::Script_Coptic,                 "Coptic" },
        // 4.0
        { QChar::Script_Limbu,                  "Limbu" },
        { QChar::Script_TaiLe,                  "TaiLe" },
        { QChar::Script_LinearB,                "LinearB" },
        { QChar::Script_Ugaritic,               "Ugaritic" },
        { QChar::Script_Shavian,                "Shavian" },
        { QChar::Script_Osmanya,                "Osmanya" },
        { QChar::Script_Cypriot,                "Cypriot" },
        { QChar::Script_Braille,                "Braille" },
        // 4.1
        { QChar::Script_Buginese,               "Buginese" },
        { QChar::Script_NewTaiLue,              "NewTaiLue" },
        { QChar::Script_Glagolitic,             "Glagolitic" },
        { QChar::Script_Tifinagh,               "Tifinagh" },
        { QChar::Script_SylotiNagri,            "SylotiNagri" },
        { QChar::Script_OldPersian,             "OldPersian" },
        { QChar::Script_Kharoshthi,             "Kharoshthi" },
        // 5.0
        { QChar::Script_Balinese,               "Balinese" },
        { QChar::Script_Cuneiform,              "Cuneiform" },
        { QChar::Script_Phoenician,             "Phoenician" },
        { QChar::Script_PhagsPa,                "PhagsPa" },
        { QChar::Script_Nko,                    "Nko" },
        // 5.1
        { QChar::Script_Sundanese,              "Sundanese" },
        { QChar::Script_Lepcha,                 "Lepcha" },
        { QChar::Script_OlChiki,                "OlChiki" },
        { QChar::Script_Vai,                    "Vai" },
        { QChar::Script_Saurashtra,             "Saurashtra" },
        { QChar::Script_KayahLi,                "KayahLi" },
        { QChar::Script_Rejang,                 "Rejang" },
        { QChar::Script_Lycian,                 "Lycian" },
        { QChar::Script_Carian,                 "Carian" },
        { QChar::Script_Lydian,                 "Lydian" },
        { QChar::Script_Cham,                   "Cham" },
        // 5.2
        { QChar::Script_TaiTham,                "TaiTham" },
        { QChar::Script_TaiViet,                "TaiViet" },
        { QChar::Script_Avestan,                "Avestan" },
        { QChar::Script_EgyptianHieroglyphs,    "EgyptianHieroglyphs" },
        { QChar::Script_Samaritan,              "Samaritan" },
        { QChar::Script_Lisu,                   "Lisu" },
        { QChar::Script_Bamum,                  "Bamum" },
        { QChar::Script_Javanese,               "Javanese" },
        { QChar::Script_MeeteiMayek,            "MeeteiMayek" },
        { QChar::Script_ImperialAramaic,        "ImperialAramaic" },
        { QChar::Script_OldSouthArabian,        "OldSouthArabian" },
        { QChar::Script_InscriptionalParthian,  "InscriptionalParthian" },
        { QChar::Script_InscriptionalPahlavi,   "InscriptionalPahlavi" },
        { QChar::Script_OldTurkic,              "OldTurkic" },
        { QChar::Script_Kaithi,                 "Kaithi" },
        // 6.0
        { QChar::Script_Batak,                  "Batak" },
        { QChar::Script_Brahmi,                 "Brahmi" },
        { QChar::Script_Mandaic,                "Mandaic" },
        // 6.1
        { QChar::Script_Chakma,                 "Chakma" },
        { QChar::Script_MeroiticCursive,        "MeroiticCursive" },
        { QChar::Script_MeroiticHieroglyphs,    "MeroiticHieroglyphs" },
        { QChar::Script_Miao,                   "Miao" },
        { QChar::Script_Sharada,                "Sharada" },
        { QChar::Script_SoraSompeng,            "SoraSompeng" },
        { QChar::Script_Takri,                  "Takri" },
        // 7.0
        { QChar::Script_CaucasianAlbanian,      "CaucasianAlbanian" },
        { QChar::Script_BassaVah,               "BassaVah" },
        { QChar::Script_Duployan,               "Duployan" },
        { QChar::Script_Elbasan,                "Elbasan" },
        { QChar::Script_Grantha,                "Grantha" },
        { QChar::Script_PahawhHmong,            "PahawhHmong" },
        { QChar::Script_Khojki,                 "Khojki" },
        { QChar::Script_LinearA,                "LinearA" },
        { QChar::Script_Mahajani,               "Mahajani" },
        { QChar::Script_Manichaean,             "Manichaean" },
        { QChar::Script_MendeKikakui,           "MendeKikakui" },
        { QChar::Script_Modi,                   "Modi" },
        { QChar::Script_Mro,                    "Mro" },
        { QChar::Script_OldNorthArabian,        "OldNorthArabian" },
        { QChar::Script_Nabataean,              "Nabataean" },
        { QChar::Script_Palmyrene,              "Palmyrene" },
        { QChar::Script_PauCinHau,              "PauCinHau" },
        { QChar::Script_OldPermic,              "OldPermic" },
        { QChar::Script_PsalterPahlavi,         "PsalterPahlavi" },
        { QChar::Script_Siddham,                "Siddham" },
        { QChar::Script_Khudawadi,              "Khudawadi" },
        { QChar::Script_Tirhuta,                "Tirhuta" },
        { QChar::Script_WarangCiti,             "WarangCiti" },
        // 8.0
        { QChar::Script_Ahom,                   "Ahom" },
        { QChar::Script_AnatolianHieroglyphs,   "AnatolianHieroglyphs" },
        { QChar::Script_Hatran,                 "Hatran" },
        { QChar::Script_Multani,                "Multani" },
        { QChar::Script_OldHungarian,           "OldHungarian" },
        { QChar::Script_SignWriting,            "SignWriting" },
        // 9.0
        { QChar::Script_Adlam,                  "Adlam" },
        { QChar::Script_Bhaiksuki,              "Bhaiksuki" },
        { QChar::Script_Marchen,                "Marchen" },
        { QChar::Script_Newa,                   "Newa" },
        { QChar::Script_Osage,                  "Osage" },
        { QChar::Script_Tangut,                 "Tangut" },
        // 10.0
        { QChar::Script_MasaramGondi,           "MasaramGondi" },
        { QChar::Script_Nushu,                  "Nushu" },
        { QChar::Script_Soyombo,                "Soyombo" },
        { QChar::Script_ZanabazarSquare,        "ZanabazarSquare" },
        // 12.1
        { QChar::Script_Dogra,                  "Dogra" },
        { QChar::Script_GunjalaGondi,           "GunjalaGondi" },
        { QChar::Script_HanifiRohingya,         "HanifiRohingya" },
        { QChar::Script_Makasar,                "Makasar" },
        { QChar::Script_Medefaidrin,            "Medefaidrin" },
        { QChar::Script_OldSogdian,             "OldSogdian" },
        { QChar::Script_Sogdian,                "Sogdian" },
        { QChar::Script_Elymaic,                "Elymaic" },
        { QChar::Script_Nandinagari,            "Nandinagari" },
        { QChar::Script_NyiakengPuachueHmong,   "NyiakengPuachueHmong" },
        { QChar::Script_Wancho,                 "Wancho" },
        // 13.0
        { QChar::Script_Chorasmian,             "Chorasmian" },
        { QChar::Script_DivesAkuru,             "DivesAkuru" },
        { QChar::Script_KhitanSmallScript,      "KhitanSmallScript" },
        { QChar::Script_Yezidi,                 "Yezidi" },

        // unhandled
        { QChar::Script_Unknown,                0 }
    };
    Scrpt *p = scripts;
    while (p->name) {
        scriptMap.insert(p->name, p->script);
        ++p;
    }
}

// Keep this one in sync with the code in createPropertyInfo
static const char *property_string =
    "enum Case {\n"
    "    LowerCase,\n"
    "    UpperCase,\n"
    "    TitleCase,\n"
    "    CaseFold,\n"
    "\n"
    "    NumCases\n"
    "};\n"
    "\n"
    "struct Properties {\n"
    "    ushort category            : 8; /* 5 used */\n"
    "    ushort direction           : 8; /* 5 used */\n"
    "    ushort combiningClass      : 8;\n"
    "    ushort joining             : 3;\n"
    "    signed short digitValue    : 5;\n"
    "    signed short mirrorDiff    : 16;\n"
    "    ushort unicodeVersion      : 8; /* 5 used */\n"
    "    ushort nfQuickCheck        : 8;\n" // could be narrowed
    "#ifdef Q_OS_WASM\n"
    "    unsigned char              : 0; //wasm 64 packing trick\n"
    "#endif\n"
    "    struct {\n"
    "        ushort special    : 1;\n"
    "        signed short diff : 15;\n"
    "    } cases[NumCases];\n"
    "#ifdef Q_OS_WASM\n"
    "    unsigned char              : 0; //wasm 64 packing trick\n"
    "#endif\n"
    "    ushort graphemeBreakClass  : 5; /* 5 used */\n"
    "    ushort wordBreakClass      : 5; /* 5 used */\n"
    "    ushort lineBreakClass      : 6; /* 6 used */\n"
    "    ushort sentenceBreakClass  : 8; /* 4 used */\n"
    "    ushort script              : 8;\n"
    "};\n\n"
    "Q_CORE_EXPORT const Properties * QT_FASTCALL properties(uint ucs4) noexcept;\n"
    "Q_CORE_EXPORT const Properties * QT_FASTCALL properties(ushort ucs2) noexcept;\n"
    "\n";

static const char *methods =
    "Q_CORE_EXPORT GraphemeBreakClass QT_FASTCALL graphemeBreakClass(uint ucs4) noexcept;\n"
    "inline GraphemeBreakClass graphemeBreakClass(QChar ch) noexcept\n"
    "{ return graphemeBreakClass(ch.unicode()); }\n"
    "\n"
    "Q_CORE_EXPORT WordBreakClass QT_FASTCALL wordBreakClass(uint ucs4) noexcept;\n"
    "inline WordBreakClass wordBreakClass(QChar ch) noexcept\n"
    "{ return wordBreakClass(ch.unicode()); }\n"
    "\n"
    "Q_CORE_EXPORT SentenceBreakClass QT_FASTCALL sentenceBreakClass(uint ucs4) noexcept;\n"
    "inline SentenceBreakClass sentenceBreakClass(QChar ch) noexcept\n"
    "{ return sentenceBreakClass(ch.unicode()); }\n"
    "\n"
    "Q_CORE_EXPORT LineBreakClass QT_FASTCALL lineBreakClass(uint ucs4) noexcept;\n"
    "inline LineBreakClass lineBreakClass(QChar ch) noexcept\n"
    "{ return lineBreakClass(ch.unicode()); }\n"
    "\n";

static const int SizeOfPropertiesStruct = 20;

static const QByteArray sizeOfPropertiesStructCheck =
        "Q_STATIC_ASSERT(sizeof(Properties) == " + QByteArray::number(SizeOfPropertiesStruct) + ");\n\n";

struct PropertyFlags {
    bool operator==(const PropertyFlags &o) const {
        return (combiningClass == o.combiningClass
                && category == o.category
                && direction == o.direction
                && joining == o.joining
                && age == o.age
                && digitValue == o.digitValue
                && mirrorDiff == o.mirrorDiff
                && lowerCaseDiff == o.lowerCaseDiff
                && upperCaseDiff == o.upperCaseDiff
                && titleCaseDiff == o.titleCaseDiff
                && caseFoldDiff == o.caseFoldDiff
                && lowerCaseSpecial == o.lowerCaseSpecial
                && upperCaseSpecial == o.upperCaseSpecial
                && titleCaseSpecial == o.titleCaseSpecial
                && caseFoldSpecial == o.caseFoldSpecial
                && graphemeBreakClass == o.graphemeBreakClass
                && wordBreakClass == o.wordBreakClass
                && sentenceBreakClass == o.sentenceBreakClass
                && lineBreakClass == o.lineBreakClass
                && script == o.script
                && nfQuickCheck == o.nfQuickCheck
            );
    }
    // from UnicodeData.txt
    uchar combiningClass : 8;
    QChar::Category category : 5;
    QChar::Direction direction : 5;
    // from ArabicShaping.txt
    QChar::JoiningType joining : 3;
    // from DerivedAge.txt
    QChar::UnicodeVersion age : 5;
    int digitValue;

    int mirrorDiff : 16;

    int lowerCaseDiff;
    int upperCaseDiff;
    int titleCaseDiff;
    int caseFoldDiff;
    bool lowerCaseSpecial;
    bool upperCaseSpecial;
    bool titleCaseSpecial;
    bool caseFoldSpecial;
    GraphemeBreakClass graphemeBreakClass;
    WordBreakClass wordBreakClass;
    SentenceBreakClass sentenceBreakClass;
    LineBreakClass lineBreakClass;
    int script;
    // from DerivedNormalizationProps.txt
    uchar nfQuickCheck;
};


static QList<int> specialCaseMap;

static int appendToSpecialCaseMap(const QList<int> &map)
{
    QList<int> utf16map;
    for (int i = 0; i < map.size(); ++i) {
        uint codepoint = map.at(i);
        // if the condition below doesn't hold anymore we need to modify our special case mapping code
        Q_ASSERT(!QChar::requiresSurrogates(codepoint));
        if (QChar::requiresSurrogates(codepoint)) {
            utf16map << QChar::highSurrogate(codepoint);
            utf16map << QChar::lowSurrogate(codepoint);
        } else {
            utf16map << codepoint;
        }
    }
    int length = utf16map.size();
    utf16map.prepend(length);

    if (specialCaseMap.isEmpty())
        specialCaseMap << 0; // placeholder

    int i = 1;
    while (i < specialCaseMap.size()) {
        int n = specialCaseMap.at(i);
        if (n == length) {
            int j;
            for (j = 1; j <= n; ++j) {
                if (specialCaseMap.at(i+j) != utf16map.at(j))
                    break;
            }
            if (j > n)
                return i;
        }
        i += n + 1;
    }

    int pos = specialCaseMap.size();
    specialCaseMap << utf16map;
    return pos;
}

// DerivedCoreProperties.txt
static inline bool isDefaultIgnorable(uint ucs4)
{
    // Default_Ignorable_Code_Point:
    //  Generated from
    //    Other_Default_Ignorable_Code_Point + Cf + Variation_Selector
    //    - White_Space - FFF9..FFFB (Annotation Characters)
    //    - 0600..0604, 06DD, 070F, 110BD (exceptional Cf characters that should be visible)
    if (ucs4 <= 0xff)
        return ucs4 == 0xad;

    return ucs4 == 0x034f
            || ucs4 == 0x061c
            || (ucs4 >= 0x115f && ucs4 <= 0x1160)
            || (ucs4 >= 0x17b4 && ucs4 <= 0x17b5)
            || (ucs4 >= 0x180b && ucs4 <= 0x180d)
            || ucs4 == 0x180e
            || (ucs4 >= 0x200b && ucs4 <= 0x200f)
            || (ucs4 >= 0x202a && ucs4 <= 0x202e)
            || (ucs4 >= 0x2060 && ucs4 <= 0x206f)
            || ucs4 == 0x3164
            || (ucs4 >= 0xfe00 && ucs4 <= 0xfe0f)
            || ucs4 == 0xfeff
            || ucs4 == 0xffa0
            || (ucs4 >= 0xfff0 && ucs4 <= 0xfff8)
            || (ucs4 >= 0x1bca0 && ucs4 <= 0x1bca3)
            || (ucs4 >= 0x1d173 && ucs4 <= 0x1d17a)
            || (ucs4 >= 0xe0000 && ucs4 <= 0xe0fff);
}

struct UnicodeData {
    UnicodeData(int codepoint = 0) {
        p.category = QChar::Other_NotAssigned; // Cn
        p.combiningClass = 0;

        p.direction = QChar::DirL;
        // DerivedBidiClass.txt
        // The unassigned code points that default to AL are in the ranges:
        //     [U+0600..U+07BF, U+08A0..U+08FF, U+FB50..U+FDCF, U+FDF0..U+FDFF, U+FE70..U+FEFF, U+1EE00..U+1EEFF]
        if ((codepoint >= 0x0600 && codepoint <= 0x07BF)
            || (codepoint >= 0x08A0 && codepoint <= 0x08FF)
            || (codepoint >= 0xFB50 && codepoint <= 0xFDCF)
            || (codepoint >= 0xFDF0 && codepoint <= 0xFDFF)
            || (codepoint >= 0xFE70 && codepoint <= 0xFEFF)
            || (codepoint >= 0x1EE00 && codepoint <= 0x1EEFF)) {
            p.direction = QChar::DirAL;
        }
        // The unassigned code points that default to R are in the ranges:
        //     [U+0590..U+05FF, U+07C0..U+089F, U+FB1D..U+FB4F, U+10800..U+10FFF, U+1E800..U+1EDFF, U+1EF00..U+1EFFF]
        else if ((codepoint >= 0x0590 && codepoint <= 0x05FF)
            || (codepoint >= 0x07C0 && codepoint <= 0x089F)
            || (codepoint >= 0xFB1D && codepoint <= 0xFB4F)
            || (codepoint >= 0x10800 && codepoint <= 0x10FFF)
            || (codepoint >= 0x1E800 && codepoint <= 0x1EDFF)
            || (codepoint >= 0x1EF00 && codepoint <= 0x1EFFF)) {
            p.direction = QChar::DirR;
        }
        // The unassigned code points that default to ET are in the range:
        //     [U+20A0..U+20CF]
        else if (codepoint >= 0x20A0 && codepoint <= 0x20CF) {
            p.direction = QChar::DirET;
        }
        // The unassigned code points that default to BN have one of the following properties:
        //     Default_Ignorable_Code_Point
        //     Noncharacter_Code_Point
        else if (QChar::isNonCharacter(codepoint) || isDefaultIgnorable(codepoint)) {
            p.direction = QChar::DirBN;
        }

        p.lineBreakClass = LineBreak_AL; // XX -> AL
        // LineBreak.txt
        // The unassigned code points that default to "ID" include ranges in the following blocks:
        //     [U+3400..U+4DBF, U+4E00..U+9FFF, U+F900..U+FAFF, U+20000..U+2A6DF, U+2A700..U+2B73F, U+2B740..U+2B81F, U+2B820..U+2CEAF, U+2F800..U+2FA1F]
        // and any other reserved code points on
        //     [U+20000..U+2FFFD, U+30000..U+3FFFD]
        if ((codepoint >= 0x3400 && codepoint <= 0x4DBF)
            || (codepoint >= 0x4E00 && codepoint <= 0x9FFF)
            || (codepoint >= 0xF900 && codepoint <= 0xFAFF)
            || (codepoint >= 0x20000 && codepoint <= 0x2A6DF)
            || (codepoint >= 0x2A700 && codepoint <= 0x2B73F)
            || (codepoint >= 0x2B740 && codepoint <= 0x2B81F)
            || (codepoint >= 0x2B820 && codepoint <= 0x2CEAF)
            || (codepoint >= 0x2F800 && codepoint <= 0x2FA1F)
            || (codepoint >= 0x20000 && codepoint <= 0x2FFFD)
            || (codepoint >= 0x30000 && codepoint <= 0x3FFFD)) {
            p.lineBreakClass = LineBreak_ID;
        }
        // The unassigned code points that default to "PR" comprise a range in the following block:
        //     [U+20A0..U+20CF]
        else if (codepoint >= 0x20A0 && codepoint <= 0x20CF) {
            p.lineBreakClass = LineBreak_PR;
        }

        mirroredChar = 0;
        decompositionType = QChar::NoDecomposition;
        p.joining = QChar::Joining_None;
        p.age = QChar::Unicode_Unassigned;
        p.mirrorDiff = 0;
        p.digitValue = -1;
        p.lowerCaseDiff = 0;
        p.upperCaseDiff = 0;
        p.titleCaseDiff = 0;
        p.caseFoldDiff = 0;
        p.lowerCaseSpecial = 0;
        p.upperCaseSpecial = 0;
        p.titleCaseSpecial = 0;
        p.caseFoldSpecial = 0;
        p.graphemeBreakClass = GraphemeBreak_Any;
        p.wordBreakClass = WordBreak_Any;
        p.sentenceBreakClass = SentenceBreak_Any;
        p.script = QChar::Script_Unknown;
        p.nfQuickCheck = 0;
        propertyIndex = -1;
        excludedComposition = false;
    }

    static UnicodeData &valueRef(int codepoint);

    PropertyFlags p;

    // from UnicodeData.txt
    QChar::Decomposition decompositionType;
    QList<int> decomposition;

    QList<int> specialFolding;

    // from BidiMirroring.txt
    int mirroredChar;

    // DerivedNormalizationProps.txt
    bool excludedComposition;

    // computed position of unicode property set
    int propertyIndex;
};

static QList<UnicodeData> unicodeData;

UnicodeData &UnicodeData::valueRef(int codepoint)
{
    static bool initialized = false;
    if (!initialized) {
        unicodeData.reserve(QChar::LastValidCodePoint + 1);
        for (int uc = 0; uc <= QChar::LastValidCodePoint; ++uc)
            unicodeData.append(UnicodeData(uc));
        initialized = true;
    }

    Q_ASSERT(codepoint <= 0x10ffff);
    return unicodeData[codepoint];
}


static QHash<int, int> decompositionLength;
static int highestComposedCharacter = 0;
static int numLigatures = 0;
static int highestLigature = 0;

struct Ligature {
    int u1;
    int u2;
    int ligature;
};
// we need them sorted after the first component for fast lookup
bool operator < (const Ligature &l1, const Ligature &l2)
{ return l1.u1 < l2.u1; }

static QHash<int, QList<Ligature> > ligatureHashes;

static QHash<int, int> combiningClassUsage;

static int maxLowerCaseDiff = 0;
static int maxUpperCaseDiff = 0;
static int maxTitleCaseDiff = 0;

static void readUnicodeData()
{
    qDebug("Reading UnicodeData.txt");

    enum UniDataFields {
        UD_Value,
        UD_Name,
        UD_Category,
        UD_CombiningClass,
        UD_BidiCategory,
        UD_Decomposition,
        UD_DecimalDigitValue,
        UD_DigitValue,
        UD_NumericValue,
        UD_Mirrored,
        UD_OldName,
        UD_Comment,
        UD_UpperCase,
        UD_LowerCase,
        UD_TitleCase
    };

    QFile f("data/UnicodeData.txt");
    if (!f.exists())
        qFatal("Couldn't find UnicodeData.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.truncate(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);
        if (line.isEmpty())
            continue;

        QList<QByteArray> properties = line.split(';');
        bool ok;
        int codepoint = properties[UD_Value].toInt(&ok, 16);
        Q_ASSERT(ok);
        Q_ASSERT(codepoint <= QChar::LastValidCodePoint);
        int lastCodepoint = codepoint;

        QByteArray name = properties[UD_Name];
        if (name.startsWith('<') && name.contains("First")) {
            QByteArray nextLine;
            nextLine.resize(1024);
            f.readLine(nextLine.data(), 1024);
            QList<QByteArray> properties = nextLine.split(';');
            Q_ASSERT(properties[UD_Name].startsWith('<') && properties[UD_Name].contains("Last"));
            lastCodepoint = properties[UD_Value].toInt(&ok, 16);
            Q_ASSERT(ok);
            Q_ASSERT(lastCodepoint <= QChar::LastValidCodePoint);
        }

        UnicodeData &data = UnicodeData::valueRef(codepoint);
        data.p.category = categoryMap.value(properties[UD_Category], QChar::Other_NotAssigned);
        data.p.combiningClass = properties[UD_CombiningClass].toInt();
        if (!combiningClassUsage.contains(data.p.combiningClass))
            combiningClassUsage[data.p.combiningClass] = 1;
        else
            ++combiningClassUsage[data.p.combiningClass];

        Direction dir = directionMap.value(properties[UD_BidiCategory], Dir_Unassigned);
        if (dir == Dir_Unassigned)
            qFatal("unhandled direction value: %s", properties[UD_BidiCategory].constData());
        data.p.direction = QChar::Direction(dir);

        if (!properties[UD_UpperCase].isEmpty()) {
            int upperCase = properties[UD_UpperCase].toInt(&ok, 16);
            Q_ASSERT(ok);
            int diff = upperCase - codepoint;
            // if the conditions below doesn't hold anymore we need to modify our upper casing code
            Q_ASSERT(QChar::requiresSurrogates(codepoint) == QChar::requiresSurrogates(upperCase));
            if (QChar::requiresSurrogates(codepoint)) {
                Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(upperCase));
                Q_ASSERT(QChar::lowSurrogate(codepoint) + diff == QChar::lowSurrogate(upperCase));
            }
            if (qAbs(diff) >= (1<<13)) {
                qWarning() << "upperCaseDiff exceeded (" << hex << codepoint << "->" << upperCase << "); map it for special case";
                data.p.upperCaseSpecial = true;
                data.p.upperCaseDiff = appendToSpecialCaseMap(QList<int>() << upperCase);
            } else {
                data.p.upperCaseDiff = diff;
                maxUpperCaseDiff = qMax(maxUpperCaseDiff, qAbs(diff));
            }
        }
        if (!properties[UD_LowerCase].isEmpty()) {
            int lowerCase = properties[UD_LowerCase].toInt(&ok, 16);
            Q_ASSERT(ok);
            int diff = lowerCase - codepoint;
            // if the conditions below doesn't hold anymore we need to modify our lower casing code
            Q_ASSERT(QChar::requiresSurrogates(codepoint) == QChar::requiresSurrogates(lowerCase));
            if (QChar::requiresSurrogates(codepoint)) {
                Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(lowerCase));
                Q_ASSERT(QChar::lowSurrogate(codepoint) + diff == QChar::lowSurrogate(lowerCase));
            }
            if (qAbs(diff) >= (1<<13)) {
                qWarning() << "lowerCaseDiff exceeded (" << hex << codepoint << "->" << lowerCase << "); map it for special case";
                data.p.lowerCaseSpecial = true;
                data.p.lowerCaseDiff = appendToSpecialCaseMap(QList<int>() << lowerCase);
            } else {
                data.p.lowerCaseDiff = diff;
                maxLowerCaseDiff = qMax(maxLowerCaseDiff, qAbs(diff));
            }
        }
        // we want toTitleCase to map to ToUpper in case we don't have any titlecase.
        if (properties[UD_TitleCase].isEmpty())
            properties[UD_TitleCase] = properties[UD_UpperCase];
        if (!properties[UD_TitleCase].isEmpty()) {
            int titleCase = properties[UD_TitleCase].toInt(&ok, 16);
            Q_ASSERT(ok);
            int diff = titleCase - codepoint;
            // if the conditions below doesn't hold anymore we need to modify our title casing code
            Q_ASSERT(QChar::requiresSurrogates(codepoint) == QChar::requiresSurrogates(titleCase));
            if (QChar::requiresSurrogates(codepoint)) {
                Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(titleCase));
                Q_ASSERT(QChar::lowSurrogate(codepoint) + diff == QChar::lowSurrogate(titleCase));
            }
            if (qAbs(diff) >= (1<<13)) {
                qWarning() << "titleCaseDiff exceeded (" << hex << codepoint << "->" << titleCase << "); map it for special case";
                data.p.titleCaseSpecial = true;
                data.p.titleCaseDiff = appendToSpecialCaseMap(QList<int>() << titleCase);
            } else {
                data.p.titleCaseDiff = diff;
                maxTitleCaseDiff = qMax(maxTitleCaseDiff, qAbs(diff));
            }
        }

        if (!properties[UD_DigitValue].isEmpty())
            data.p.digitValue = properties[UD_DigitValue].toInt();

        // decompositition
        QByteArray decomposition = properties[UD_Decomposition];
        if (!decomposition.isEmpty()) {
            highestComposedCharacter = qMax(highestComposedCharacter, codepoint);
            QList<QByteArray> d = decomposition.split(' ');
            if (d[0].contains('<')) {
                data.decompositionType = decompositionMap.value(d[0], QChar::NoDecomposition);
                if (data.decompositionType == QChar::NoDecomposition)
                    qFatal("unhandled decomposition type: %s", d[0].constData());
                d.takeFirst();
            } else {
                data.decompositionType = QChar::Canonical;
            }
            for (int i = 0; i < d.size(); ++i) {
                data.decomposition.append(d[i].toInt(&ok, 16));
                Q_ASSERT(ok);
            }
            ++decompositionLength[data.decomposition.size()];
        }

        for (int i = codepoint; i <= lastCodepoint; ++i)
            unicodeData[i] = data;
    }
}

static int maxMirroredDiff = 0;

static void readBidiMirroring()
{
    qDebug("Reading BidiMirroring.txt");

    QFile f("data/BidiMirroring.txt");
    if (!f.exists())
        qFatal("Couldn't find BidiMirroring.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);

        if (line.isEmpty())
            continue;
        line = line.replace(" ", "");

        QList<QByteArray> pair = line.split(';');
        Q_ASSERT(pair.size() == 2);

        bool ok;
        int codepoint = pair[0].toInt(&ok, 16);
        Q_ASSERT(ok);
        int mirror = pair[1].toInt(&ok, 16);
        Q_ASSERT(ok);

        UnicodeData &d = UnicodeData::valueRef(codepoint);
        d.mirroredChar = mirror;
        d.p.mirrorDiff = d.mirroredChar - codepoint;
        maxMirroredDiff = qMax(maxMirroredDiff, qAbs(d.p.mirrorDiff));
    }
}

static void readArabicShaping()
{
    qDebug("Reading ArabicShaping.txt");

    // Initialize defaults:
    // Code points that are not explicitly listed in ArabicShaping.txt are either of joining type T or U:
    // - Those that not explicitly listed that are of General Category Mn, Me, or Cf have joining type T.
    // - All others not explicitly listed have joining type U.
    for (int codepoint = 0; codepoint <= QChar::LastValidCodePoint; ++codepoint) {
        UnicodeData &d = UnicodeData::valueRef(codepoint);
        if (d.p.joining == QChar::Joining_None) {
            if (d.p.category == QChar::Mark_NonSpacing || d.p.category == QChar::Mark_Enclosing || d.p.category == QChar::Other_Format)
                d.p.joining = QChar::Joining_Transparent;
        }
    }

    QFile f("data/ArabicShaping.txt");
    if (!f.exists())
        qFatal("Couldn't find ArabicShaping.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);
        line = line.trimmed();

        if (line.isEmpty())
            continue;

        QList<QByteArray> l = line.split(';');
        Q_ASSERT(l.size() == 4);

        bool ok;
        int codepoint = l[0].toInt(&ok, 16);
        Q_ASSERT(ok);

        UnicodeData &d = UnicodeData::valueRef(codepoint);
        JoiningType joining = joining_map.value(l[2].trimmed(), Joining_Unassigned);
        switch (joining) {
        case Joining_Unassigned:
            qFatal("%x: unassigned or unhandled joining type: %s", codepoint, l[2].constData());
            break;
        case Joining_Transparent:
            switch (d.p.category) {
            case QChar::Mark_Enclosing:
            case QChar::Mark_NonSpacing:
            case QChar::Letter_Modifier:
            case QChar::Other_Format:
                break;
            default:
                qFatal("%x: joining type '%s' was met (category: %d); "
                       "the current implementation needs to be revised!",
                       codepoint, l[2].constData(), d.p.category);
            }
            Q_FALLTHROUGH();
        default:
            d.p.joining = QChar::JoiningType(joining);
            break;
        }
    }
}

static void readDerivedAge()
{
    qDebug("Reading DerivedAge.txt");

    QFile f("data/DerivedAge.txt");
    if (!f.exists())
        qFatal("Couldn't find DerivedAge.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);
        line.replace(" ", "");

        if (line.isEmpty())
            continue;

        QList<QByteArray> l = line.split(';');
        Q_ASSERT(l.size() == 2);

        QByteArray codes = l[0];
        codes.replace("..", ".");
        QList<QByteArray> cl = codes.split('.');

        bool ok;
        int from = cl[0].toInt(&ok, 16);
        Q_ASSERT(ok);
        int to = from;
        if (cl.size() == 2) {
            to = cl[1].toInt(&ok, 16);
            Q_ASSERT(ok);
        }

        QChar::UnicodeVersion age = age_map.value(l[1].trimmed(), QChar::Unicode_Unassigned);
        //qDebug() << hex << from << ".." << to << ba << age;
        if (age == QChar::Unicode_Unassigned)
            qFatal("unassigned or unhandled age value: %s", l[1].constData());

        for (int codepoint = from; codepoint <= to; ++codepoint) {
            UnicodeData &d = UnicodeData::valueRef(codepoint);
            d.p.age = age;
        }
    }
}

static void readDerivedNormalizationProps()
{
    qDebug("Reading DerivedNormalizationProps.txt");

    QFile f("data/DerivedNormalizationProps.txt");
    if (!f.exists())
        qFatal("Couldn't find DerivedNormalizationProps.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);

        if (line.trimmed().isEmpty())
            continue;

        QList<QByteArray> l = line.split(';');
        Q_ASSERT(l.size() >= 2);

        QByteArray propName = l[1].trimmed();
        if (propName != "Full_Composition_Exclusion" &&
            propName != "NFD_QC" && propName != "NFC_QC" &&
            propName != "NFKD_QC" && propName != "NFKC_QC") {
            // ###
            continue;
        }

        QByteArray codes = l[0].trimmed();
        codes.replace("..", ".");
        QList<QByteArray> cl = codes.split('.');

        bool ok;
        int from = cl[0].toInt(&ok, 16);
        Q_ASSERT(ok);
        int to = from;
        if (cl.size() == 2) {
            to = cl[1].toInt(&ok, 16);
            Q_ASSERT(ok);
        }

        for (int codepoint = from; codepoint <= to; ++codepoint) {
            UnicodeData &d = UnicodeData::valueRef(codepoint);
            if (propName == "Full_Composition_Exclusion") {
                d.excludedComposition = true;
            } else {
                Q_STATIC_ASSERT(QString::NormalizationForm_D == 0);
                Q_STATIC_ASSERT(QString::NormalizationForm_C == 1);
                Q_STATIC_ASSERT(QString::NormalizationForm_KD == 2);
                Q_STATIC_ASSERT(QString::NormalizationForm_KC == 3);

                QString::NormalizationForm form;
                if (propName == "NFD_QC")
                    form = QString::NormalizationForm_D;
                else if (propName == "NFC_QC")
                    form = QString::NormalizationForm_C;
                else if (propName == "NFKD_QC")
                    form = QString::NormalizationForm_KD;
                else// if (propName == "NFKC_QC")
                    form = QString::NormalizationForm_KC;

                Q_ASSERT(l.size() == 3);
                l[2] = l[2].trimmed();

                enum { NFQC_YES = 0, NFQC_NO = 1, NFQC_MAYBE = 3 };
                uchar ynm = (l[2] == "N" ? NFQC_NO : l[2] == "M" ? NFQC_MAYBE : NFQC_YES);
                if (ynm == NFQC_MAYBE) {
                    // if this changes, we need to revise the normalizationQuickCheckHelper() implementation
                    Q_ASSERT(form == QString::NormalizationForm_C || form == QString::NormalizationForm_KC);
                }
                d.p.nfQuickCheck |= (ynm << (form << 1)); // 2 bits per NF
            }
        }
    }

    for (int codepoint = 0; codepoint <= QChar::LastValidCodePoint; ++codepoint) {
        UnicodeData &d = UnicodeData::valueRef(codepoint);
        if (!d.excludedComposition
            && d.decompositionType == QChar::Canonical
            && d.decomposition.size() > 1) {
            Q_ASSERT(d.decomposition.size() == 2);

            int part1 = d.decomposition.at(0);
            int part2 = d.decomposition.at(1);

            // all non-starters are listed in DerivedNormalizationProps.txt
            // and already excluded from composition
            Q_ASSERT(UnicodeData::valueRef(part1).p.combiningClass == 0);

            ++numLigatures;
            highestLigature = qMax(highestLigature, part1);
            Ligature l = { part1, part2, codepoint };
            ligatureHashes[part2].append(l);
        }
    }
}


struct NormalizationCorrection {
    uint codepoint;
    uint mapped;
    int version;
};

static QByteArray createNormalizationCorrections()
{
    qDebug("Reading NormalizationCorrections.txt");

    QFile f("data/NormalizationCorrections.txt");
    if (!f.exists())
        qFatal("Couldn't find NormalizationCorrections.txt");

    f.open(QFile::ReadOnly);

    QByteArray out;

    out += "struct NormalizationCorrection {\n"
           "    uint ucs4;\n"
           "    uint old_mapping;\n"
           "    int version;\n"
           "};\n\n"

           "static const NormalizationCorrection uc_normalization_corrections[] = {\n";

    int maxVersion = 0;
    int numCorrections = 0;
    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);
        line.replace(" ", "");

        if (line.isEmpty())
            continue;

        Q_ASSERT(!line.contains(".."));

        QList<QByteArray> fields = line.split(';');
        Q_ASSERT(fields.size() == 4);

        NormalizationCorrection c = { 0, 0, 0 };
        bool ok;
        c.codepoint = fields.at(0).toInt(&ok, 16);
        Q_ASSERT(ok);
        c.mapped = fields.at(1).toInt(&ok, 16);
        Q_ASSERT(ok);
        if (fields.at(3) == "3.2.0")
            c.version = QChar::Unicode_3_2;
        else if (fields.at(3) == "4.0.0")
            c.version = QChar::Unicode_4_0;
        else
            qFatal("unknown unicode version in NormalizationCorrection.txt");

        out += "    { 0x" + QByteArray::number(c.codepoint, 16) + ", 0x" + QByteArray::number(c.mapped, 16)
             + ", " + QString::number(c.version) + " },\n";
        ++numCorrections;
        maxVersion = qMax(c.version, maxVersion);
    }
    if (out.endsWith(",\n"))
        out.chop(2);

    out += "\n};\n\n"

           "enum { NumNormalizationCorrections = " + QByteArray::number(numCorrections) + " };\n"
           "enum { NormalizationCorrectionsVersionMax = " + QByteArray::number(maxVersion) + " };\n\n";

    return out;
}

static void readLineBreak()
{
    qDebug("Reading LineBreak.txt");

    QFile f("data/LineBreak.txt");
    if (!f.exists())
        qFatal("Couldn't find LineBreak.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);
        line.replace(" ", "");

        if (line.isEmpty())
            continue;

        QList<QByteArray> l = line.split(';');
        Q_ASSERT(l.size() == 2);

        QByteArray codes = l[0];
        codes.replace("..", ".");
        QList<QByteArray> cl = codes.split('.');

        bool ok;
        int from = cl[0].toInt(&ok, 16);
        Q_ASSERT(ok);
        int to = from;
        if (cl.size() == 2) {
            to = cl[1].toInt(&ok, 16);
            Q_ASSERT(ok);
        }

        LineBreakClass lb = line_break_map.value(l[1], LineBreak_Unassigned);
        if (lb == LineBreak_Unassigned)
            qFatal("unassigned line break class: %s", l[1].constData());

        for (int codepoint = from; codepoint <= to; ++codepoint) {
            UnicodeData &d = UnicodeData::valueRef(codepoint);
            d.p.lineBreakClass = lb;
        }
    }
}

static void readSpecialCasing()
{
    qDebug("Reading SpecialCasing.txt");

    QFile f("data/SpecialCasing.txt");
    if (!f.exists())
        qFatal("Couldn't find SpecialCasing.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);

        if (line.isEmpty())
            continue;

        QList<QByteArray> l = line.split(';');

        QByteArray condition = l.size() < 5 ? QByteArray() : l[4].trimmed();
        if (!condition.isEmpty())
            // #####
            continue;

        bool ok;
        int codepoint = l[0].trimmed().toInt(&ok, 16);
        Q_ASSERT(ok);

        // if the condition below doesn't hold anymore we need to modify our
        // lower/upper/title casing code and case folding code
        Q_ASSERT(!QChar::requiresSurrogates(codepoint));

//         qDebug() << "codepoint" << hex << codepoint;
//         qDebug() << line;

        QList<QByteArray> lower = l[1].trimmed().split(' ');
        QList<int> lowerMap;
        for (int i = 0; i < lower.size(); ++i) {
            bool ok;
            lowerMap.append(lower.at(i).toInt(&ok, 16));
            Q_ASSERT(ok);
        }

        QList<QByteArray> title = l[2].trimmed().split(' ');
        QList<int> titleMap;
        for (int i = 0; i < title.size(); ++i) {
            bool ok;
            titleMap.append(title.at(i).toInt(&ok, 16));
            Q_ASSERT(ok);
        }

        QList<QByteArray> upper = l[3].trimmed().split(' ');
        QList<int> upperMap;
        for (int i = 0; i < upper.size(); ++i) {
            bool ok;
            upperMap.append(upper.at(i).toInt(&ok, 16));
            Q_ASSERT(ok);
        }


        UnicodeData &ud = UnicodeData::valueRef(codepoint);
        Q_ASSERT(lowerMap.size() > 1 || lowerMap.at(0) == codepoint + ud.p.lowerCaseDiff);
        Q_ASSERT(titleMap.size() > 1 || titleMap.at(0) == codepoint + ud.p.titleCaseDiff);
        Q_ASSERT(upperMap.size() > 1 || upperMap.at(0) == codepoint + ud.p.upperCaseDiff);

        if (lowerMap.size() > 1) {
            ud.p.lowerCaseSpecial = true;
            ud.p.lowerCaseDiff = appendToSpecialCaseMap(lowerMap);
        }
        if (titleMap.size() > 1) {
            ud.p.titleCaseSpecial = true;
            ud.p.titleCaseDiff = appendToSpecialCaseMap(titleMap);
        }
        if (upperMap.size() > 1) {
            ud.p.upperCaseSpecial = true;
            ud.p.upperCaseDiff = appendToSpecialCaseMap(upperMap);
        }
    }
}

static int maxCaseFoldDiff = 0;

static void readCaseFolding()
{
    qDebug("Reading CaseFolding.txt");

    QFile f("data/CaseFolding.txt");
    if (!f.exists())
        qFatal("Couldn't find CaseFolding.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);

        if (line.isEmpty())
            continue;

        QList<QByteArray> l = line.split(';');

        bool ok;
        int codepoint = l[0].trimmed().toInt(&ok, 16);
        Q_ASSERT(ok);


        l[1] = l[1].trimmed();
        if (l[1] == "F" || l[1] == "T")
            continue;

//         qDebug() << "codepoint" << hex << codepoint;
//         qDebug() << line;
        QList<QByteArray> fold = l[2].trimmed().split(' ');
        QList<int> foldMap;
        for (int i = 0; i < fold.size(); ++i) {
            bool ok;
            foldMap.append(fold.at(i).toInt(&ok, 16));
            Q_ASSERT(ok);
        }

        UnicodeData &ud = UnicodeData::valueRef(codepoint);
        if (foldMap.size() == 1) {
            int caseFolded = foldMap.at(0);
            int diff = caseFolded - codepoint;
            // if the conditions below doesn't hold anymore we need to modify our case folding code
            Q_ASSERT(QChar::requiresSurrogates(codepoint) == QChar::requiresSurrogates(caseFolded));
            if (QChar::requiresSurrogates(codepoint)) {
                Q_ASSERT(QChar::highSurrogate(codepoint) == QChar::highSurrogate(caseFolded));
                Q_ASSERT(QChar::lowSurrogate(codepoint) + diff == QChar::lowSurrogate(caseFolded));
            }
            if (qAbs(diff) >= (1<<13)) {
                qWarning() << "caseFoldDiff exceeded (" << hex << codepoint << "->" << caseFolded << "); map it for special case";
                ud.p.caseFoldSpecial = true;
                ud.p.caseFoldDiff = appendToSpecialCaseMap(foldMap);
            } else {
                ud.p.caseFoldDiff = diff;
                maxCaseFoldDiff = qMax(maxCaseFoldDiff, qAbs(diff));
            }
        } else {
            qFatal("we currently don't support full case foldings");
//             qDebug() << "special" << hex << foldMap;
            ud.p.caseFoldSpecial = true;
            ud.p.caseFoldDiff = appendToSpecialCaseMap(foldMap);
        }
    }
}

static void readGraphemeBreak()
{
    qDebug("Reading GraphemeBreakProperty.txt");

    QFile f("data/GraphemeBreakProperty.txt");
    if (!f.exists())
        qFatal("Couldn't find GraphemeBreakProperty.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);
        line.replace(" ", "");

        if (line.isEmpty())
            continue;

        QList<QByteArray> l = line.split(';');
        Q_ASSERT(l.size() == 2);

        QByteArray codes = l[0];
        codes.replace("..", ".");
        QList<QByteArray> cl = codes.split('.');

        bool ok;
        int from = cl[0].toInt(&ok, 16);
        Q_ASSERT(ok);
        int to = from;
        if (cl.size() == 2) {
            to = cl[1].toInt(&ok, 16);
            Q_ASSERT(ok);
        }

        GraphemeBreakClass brk = grapheme_break_map.value(l[1], GraphemeBreak_Unassigned);
        if (brk == GraphemeBreak_Unassigned)
            qFatal("unassigned grapheme break class: %s", l[1].constData());

        for (int codepoint = from; codepoint <= to; ++codepoint) {
            UnicodeData &ud = UnicodeData::valueRef(codepoint);
            ud.p.graphemeBreakClass = brk;
        }
    }
}

static void readWordBreak()
{
    qDebug("Reading WordBreakProperty.txt");

    QFile f("data/WordBreakProperty.txt");
    if (!f.exists())
        qFatal("Couldn't find WordBreakProperty.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);
        line.replace(" ", "");

        if (line.isEmpty())
            continue;

        QList<QByteArray> l = line.split(';');
        Q_ASSERT(l.size() == 2);

        QByteArray codes = l[0];
        codes.replace("..", ".");
        QList<QByteArray> cl = codes.split('.');

        bool ok;
        int from = cl[0].toInt(&ok, 16);
        Q_ASSERT(ok);
        int to = from;
        if (cl.size() == 2) {
            to = cl[1].toInt(&ok, 16);
            Q_ASSERT(ok);
        }

        WordBreakClass brk = word_break_map.value(l[1], WordBreak_Unassigned);
        if (brk == WordBreak_Unassigned)
            qFatal("unassigned word break class: %s", l[1].constData());

        for (int codepoint = from; codepoint <= to; ++codepoint) {
            // ### [
            // as of Unicode 5.1, some punctuation marks were mapped to MidLetter and MidNumLet
            // which caused "hi.there" to be treated like if it were just a single word;
            // until we have a tailoring mechanism, retain the old behavior by remapping those characters here.
            if (codepoint == 0x002E) // FULL STOP
                brk = WordBreak_MidNum;
            else if (codepoint == 0x003A) // COLON
                brk = WordBreak_Any;
            // ] ###
            UnicodeData &ud = UnicodeData::valueRef(codepoint);
            ud.p.wordBreakClass = brk;
        }
    }
}

static void readSentenceBreak()
{
    qDebug("Reading SentenceBreakProperty.txt");

    QFile f("data/SentenceBreakProperty.txt");
    if (!f.exists())
        qFatal("Couldn't find SentenceBreakProperty.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line;
        line.resize(1024);
        int len = f.readLine(line.data(), 1024);
        line.resize(len-1);

        int comment = line.indexOf('#');
        if (comment >= 0)
            line = line.left(comment);
        line.replace(" ", "");

        if (line.isEmpty())
            continue;

        QList<QByteArray> l = line.split(';');
        Q_ASSERT(l.size() == 2);

        QByteArray codes = l[0];
        codes.replace("..", ".");
        QList<QByteArray> cl = codes.split('.');

        bool ok;
        int from = cl[0].toInt(&ok, 16);
        Q_ASSERT(ok);
        int to = from;
        if (cl.size() == 2) {
            to = cl[1].toInt(&ok, 16);
            Q_ASSERT(ok);
        }

        SentenceBreakClass brk = sentence_break_map.value(l[1], SentenceBreak_Unassigned);
        if (brk == SentenceBreak_Unassigned)
            qFatal("unassigned sentence break class: %s", l[1].constData());

        for (int codepoint = from; codepoint <= to; ++codepoint) {
            UnicodeData &ud = UnicodeData::valueRef(codepoint);
            ud.p.sentenceBreakClass = brk;
        }
    }
}

#if 0
// this piece of code does full case folding and comparison. We currently
// don't use it, since this gives lots of issues with things as case insensitive
// search and replace.
static inline void foldCase(uint ch, ushort *out)
{
    const QUnicodeTables::Properties *p = qGetProp(ch);
    if (!p->caseFoldSpecial) {
        *(out++) = ch + p->caseFoldDiff;
    } else {
        const ushort *folded = specialCaseMap + p->caseFoldDiff;
        ushort length = *folded++;
        while (length--)
            *out++ = *folded++;
    }
    *out = 0;
}

static int ucstricmp(const ushort *a, const ushort *ae, const ushort *b, const ushort *be)
{
    if (a == b)
        return 0;
    if (a == 0)
        return 1;
    if (b == 0)
        return -1;

    while (a != ae && b != be) {
        const QUnicodeTables::Properties *pa = qGetProp(*a);
        const QUnicodeTables::Properties *pb = qGetProp(*b);
        if (pa->caseFoldSpecial | pb->caseFoldSpecial)
            goto special;
            int diff = (int)(*a + pa->caseFoldDiff) - (int)(*b + pb->caseFoldDiff);
        if ((diff))
            return diff;
        ++a;
        ++b;
        }
    }
    if (a == ae) {
        if (b == be)
            return 0;
        return -1;
    }
    return 1;
special:
    ushort abuf[SPECIAL_CASE_MAX_LEN + 1];
    ushort bbuf[SPECIAL_CASE_MAX_LEN + 1];
    abuf[0] = bbuf[0] = 0;
    ushort *ap = abuf;
    ushort *bp = bbuf;
    while (1) {
        if (!*ap) {
            if (a == ae) {
                if (!*bp && b == be)
                    return 0;
                return -1;
            }
            foldCase(*(a++), abuf);
            ap = abuf;
        }
        if (!*bp) {
            if (b == be)
                return 1;
            foldCase(*(b++), bbuf);
            bp = bbuf;
        }
        if (*ap != *bp)
            return (int)*ap - (int)*bp;
        ++ap;
        ++bp;
    }
}


static int ucstricmp(const ushort *a, const ushort *ae, const uchar *b)
{
    if (a == 0)
        return 1;
    if (b == 0)
        return -1;

    while (a != ae && *b) {
        const QUnicodeTables::Properties *pa = qGetProp(*a);
        const QUnicodeTables::Properties *pb = qGetProp((ushort)*b);
        if (pa->caseFoldSpecial | pb->caseFoldSpecial)
            goto special;
        int diff = (int)(*a + pa->caseFoldDiff) - (int)(*b + pb->caseFoldDiff);
        if ((diff))
            return diff;
        ++a;
        ++b;
    }
    if (a == ae) {
        if (!*b)
            return 0;
        return -1;
    }
    return 1;

special:
    ushort abuf[SPECIAL_CASE_MAX_LEN + 1];
    ushort bbuf[SPECIAL_CASE_MAX_LEN + 1];
    abuf[0] = bbuf[0] = 0;
    ushort *ap = abuf;
    ushort *bp = bbuf;
    while (1) {
        if (!*ap) {
            if (a == ae) {
                if (!*bp && !*b)
                    return 0;
                return -1;
            }
            foldCase(*(a++), abuf);
            ap = abuf;
        }
        if (!*bp) {
            if (!*b)
                return 1;
            foldCase(*(b++), bbuf);
            bp = bbuf;
        }
        if (*ap != *bp)
            return (int)*ap - (int)*bp;
        ++ap;
        ++bp;
    }
}
#endif

#if 0
static QList<QByteArray> blockNames;
struct BlockInfo
{
    int blockIndex;
    int firstCodePoint;
    int lastCodePoint;
};
static QList<BlockInfo> blockInfoList;

static void readBlocks()
{
    qDebug("Reading Blocks.txt");

    QFile f("data/Blocks.txt");
    if (!f.exists())
        qFatal("Couldn't find Blocks.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line = f.readLine();
        line.resize(line.size() - 1);

        int comment = line.indexOf("#");
        if (comment >= 0)
            line = line.left(comment);

        line.replace(" ", "");

        if (line.isEmpty())
            continue;

        int semicolon = line.indexOf(';');
        Q_ASSERT(semicolon >= 0);
        QByteArray codePoints = line.left(semicolon);
        QByteArray blockName = line.mid(semicolon + 1);

        int blockIndex = blockNames.indexOf(blockName);
        if (blockIndex == -1) {
            blockIndex = blockNames.size();
            blockNames.append(blockName);
        }

        codePoints.replace("..", ".");
        QList<QByteArray> cl = codePoints.split('.');

        bool ok;
        int first = cl[0].toInt(&ok, 16);
        Q_ASSERT(ok);
        int last = first;
        if (cl.size() == 2) {
            last = cl[1].toInt(&ok, 16);
            Q_ASSERT(ok);
        }

        BlockInfo blockInfo = { blockIndex, first, last };
        blockInfoList.append(blockInfo);
    }
}
#endif

static void readScripts()
{
    qDebug("Reading Scripts.txt");

    QFile f("data/Scripts.txt");
    if (!f.exists())
        qFatal("Couldn't find Scripts.txt");

    f.open(QFile::ReadOnly);

    while (!f.atEnd()) {
        QByteArray line = f.readLine();
        line.resize(line.size() - 1);

        int comment = line.indexOf("#");
        if (comment >= 0)
            line = line.left(comment);

        line.replace(" ", "");
        line.replace("_", "");

        if (line.isEmpty())
            continue;

        int semicolon = line.indexOf(';');
        Q_ASSERT(semicolon >= 0);
        QByteArray codePoints = line.left(semicolon);
        QByteArray scriptName = line.mid(semicolon + 1);

        codePoints.replace("..", ".");
        QList<QByteArray> cl = codePoints.split('.');

        bool ok;
        int first = cl[0].toInt(&ok, 16);
        Q_ASSERT(ok);
        int last = first;
        if (cl.size() == 2) {
            last = cl[1].toInt(&ok, 16);
            Q_ASSERT(ok);
        }

        if (!scriptMap.contains(scriptName))
            qFatal("Unhandled script property value: %s", scriptName.constData());
        QChar::Script script = scriptMap.value(scriptName, QChar::Script_Unknown);

        for (int codepoint = first; codepoint <= last; ++codepoint) {
            UnicodeData &ud = UnicodeData::valueRef(codepoint);
            ud.p.script = script;
        }
    }
}

#if 0
static void dump(int from, int to)
{
    for (int i = from; i <= to; ++i) {
        UnicodeData &d = UnicodeData::valueRef(i);
        qDebug("0x%04x: cat=%d combining=%d dir=%d case=%x mirror=%x joining=%d age=%d",
               i, d.p.category, d.p.combiningClass, d.p.direction, d.otherCase, d.mirroredChar, d.p.joining, d.p.age);
        if (d.decompositionType != QChar::NoDecomposition) {
            qDebug("    decomposition: type=%d, length=%d, first=%x", d.decompositionType, d.decomposition.size(),
                   d.decomposition[0]);
        }
    }
    qDebug(" ");
}
#endif

static QList<PropertyFlags> uniqueProperties;

static void computeUniqueProperties()
{
    qDebug("computeUniqueProperties:");
    for (int codepoint = 0; codepoint <= QChar::LastValidCodePoint; ++codepoint) {
        UnicodeData &d = UnicodeData::valueRef(codepoint);
        int index = uniqueProperties.indexOf(d.p);
        if (index == -1) {
            index = uniqueProperties.size();
            uniqueProperties.append(d.p);
        }
        d.propertyIndex = index;
    }
    qDebug("    %d unique unicode properties found", uniqueProperties.size());
}

struct UniqueBlock {
    inline UniqueBlock() : index(-1) {}

    inline bool operator==(const UniqueBlock &other) const
    { return values == other.values; }

    int index;
    QVector<int> values;
};

static QByteArray createPropertyInfo()
{
    qDebug("createPropertyInfo:");

    // we reserve one bit more than in the assert below for the sign
    Q_ASSERT(maxMirroredDiff < (1<<12));
    Q_ASSERT(maxLowerCaseDiff < (1<<13));
    Q_ASSERT(maxUpperCaseDiff < (1<<13));
    Q_ASSERT(maxTitleCaseDiff < (1<<13));
    Q_ASSERT(maxCaseFoldDiff < (1<<13));

    const int BMP_BLOCKSIZE = 32;
    const int BMP_SHIFT = 5;
    const int BMP_END = 0x11000;
    const int SMP_END = 0x110000;
    const int SMP_BLOCKSIZE = 256;
    const int SMP_SHIFT = 8;

    QList<UniqueBlock> uniqueBlocks;
    QVector<int> blockMap;
    int used = 0;

    for (int block = 0; block < BMP_END/BMP_BLOCKSIZE; ++block) {
        UniqueBlock b;
        b.values.reserve(BMP_BLOCKSIZE);
        for (int i = 0; i < BMP_BLOCKSIZE; ++i) {
            int uc = block*BMP_BLOCKSIZE + i;
            UnicodeData &d = UnicodeData::valueRef(uc);
            b.values.append(d.propertyIndex);
        }
        int index = uniqueBlocks.indexOf(b);
        if (index == -1) {
            index = uniqueBlocks.size();
            b.index = used;
            used += BMP_BLOCKSIZE;
            uniqueBlocks.append(b);
        }
        blockMap.append(uniqueBlocks.at(index).index);
    }
    int bmp_blocks = uniqueBlocks.size();

    for (int block = BMP_END/SMP_BLOCKSIZE; block < SMP_END/SMP_BLOCKSIZE; ++block) {
        UniqueBlock b;
        b.values.reserve(SMP_BLOCKSIZE);
        for (int i = 0; i < SMP_BLOCKSIZE; ++i) {
            int uc = block*SMP_BLOCKSIZE + i;
            UnicodeData &d = UnicodeData::valueRef(uc);
            b.values.append(d.propertyIndex);
        }
        int index = uniqueBlocks.indexOf(b);
        if (index == -1) {
            index = uniqueBlocks.size();
            b.index = used;
            used += SMP_BLOCKSIZE;
            uniqueBlocks.append(b);
        }
        blockMap.append(uniqueBlocks.at(index).index);
    }
    int smp_blocks = uniqueBlocks.size() - bmp_blocks;

    int bmp_block_data = bmp_blocks*BMP_BLOCKSIZE*sizeof(unsigned short);
    int bmp_trie = BMP_END/BMP_BLOCKSIZE*sizeof(unsigned short);
    int bmp_mem = bmp_block_data + bmp_trie;
    qDebug("    %d unique blocks in BMP.", bmp_blocks);
    qDebug("        block data uses: %d bytes", bmp_block_data);
    qDebug("        trie data uses : %d bytes", bmp_trie);

    int smp_block_data = smp_blocks*SMP_BLOCKSIZE*sizeof(unsigned short);
    int smp_trie = (SMP_END-BMP_END)/SMP_BLOCKSIZE*sizeof(unsigned short);
    int smp_mem = smp_block_data + smp_trie;
    qDebug("    %d unique blocks in SMP.", smp_blocks);
    qDebug("        block data uses: %d bytes", smp_block_data);
    qDebug("        trie data uses : %d bytes", smp_trie);

    int prop_data = uniqueProperties.size() * SizeOfPropertiesStruct;
    qDebug("\n        properties data uses : %d bytes", prop_data);
    qDebug("    memory usage: %d bytes", bmp_mem + smp_mem + prop_data);

    Q_ASSERT(blockMap.last() + blockMap.size() < (1<<(sizeof(unsigned short)*8)));

    QByteArray out;

    out += "static const unsigned short uc_property_trie[] = {\n";
    // first write the map
    out += "    // [0x0..0x" + QByteArray::number(BMP_END, 16) + ")";
    for (int i = 0; i < BMP_END/BMP_BLOCKSIZE; ++i) {
        if (!(i % 8)) {
            if (out.endsWith(' '))
                out.chop(1);
            if (!((i*BMP_BLOCKSIZE) % 0x1000))
                out += "\n";
            out += "\n    ";
        }
        out += QByteArray::number(blockMap.at(i) + blockMap.size());
        out += ", ";
    }
    if (out.endsWith(' '))
        out.chop(1);
    out += "\n\n    // [0x" + QByteArray::number(BMP_END, 16) + "..0x" + QByteArray::number(SMP_END, 16) + ")\n";
    for (int i = BMP_END/BMP_BLOCKSIZE; i < blockMap.size(); ++i) {
        if (!(i % 8)) {
            if (out.endsWith(' '))
                out.chop(1);
            if (!(i % (0x10000/SMP_BLOCKSIZE)))
                out += "\n";
            out += "\n    ";
        }
        out += QByteArray::number(blockMap.at(i) + blockMap.size());
        out += ", ";
    }
    if (out.endsWith(' '))
        out.chop(1);
    out += "\n";
    // write the data
    for (int i = 0; i < uniqueBlocks.size(); ++i) {
        if (out.endsWith(' '))
            out.chop(1);
        out += "\n";
        const UniqueBlock &b = uniqueBlocks.at(i);
        for (int j = 0; j < b.values.size(); ++j) {
            if (!(j % 8)) {
                if (out.endsWith(' '))
                    out.chop(1);
                out += "\n    ";
            }
            out += QByteArray::number(b.values.at(j));
            out += ", ";
        }
    }
    if (out.endsWith(", "))
        out.chop(2);
    out += "\n};\n\n";

    out += "#define GET_PROP_INDEX(ucs4) \\\n"
           "       (ucs4 < 0x" + QByteArray::number(BMP_END, 16) + " \\\n"
           "        ? (uc_property_trie[uc_property_trie[ucs4>>" + QByteArray::number(BMP_SHIFT) +
           "] + (ucs4 & 0x" + QByteArray::number(BMP_BLOCKSIZE-1, 16)+ ")]) \\\n"
           "        : (uc_property_trie[uc_property_trie[((ucs4 - 0x" + QByteArray::number(BMP_END, 16) +
           ")>>" + QByteArray::number(SMP_SHIFT) + ") + 0x" + QByteArray::number(BMP_END/BMP_BLOCKSIZE, 16) + "]"
           " + (ucs4 & 0x" + QByteArray::number(SMP_BLOCKSIZE-1, 16) + ")]))\n\n"
           "#define GET_PROP_INDEX_UCS2(ucs2) \\\n"
           "       (uc_property_trie[uc_property_trie[ucs2>>" + QByteArray::number(BMP_SHIFT) +
           "] + (ucs2 & 0x" + QByteArray::number(BMP_BLOCKSIZE-1, 16)+ ")])\n\n";

    out += "static const Properties uc_properties[] = {";
    // keep in sync with the property declaration
    for (int i = 0; i < uniqueProperties.size(); ++i) {
        const PropertyFlags &p = uniqueProperties.at(i);
        out += "\n    { ";
//     "        ushort category            : 8; /* 5 used */\n"
        out += QByteArray::number( p.category );
        out += ", ";
//     "        ushort direction           : 8; /* 5 used */\n"
        out += QByteArray::number( p.direction );
        out += ", ";
//     "        ushort combiningClass      : 8;\n"
        out += QByteArray::number( p.combiningClass );
        out += ", ";
//     "        ushort joining             : 3;\n"
        out += QByteArray::number( p.joining );
        out += ", ";
//     "        signed short digitValue    : 5;\n"
        out += QByteArray::number( p.digitValue );
        out += ", ";
//     "        signed short mirrorDiff    : 16;\n"
        out += QByteArray::number( p.mirrorDiff );
        out += ", ";
//     "        ushort unicodeVersion      : 8; /* 5 used */\n"
        out += QByteArray::number( p.age );
        out += ", ";
//     "        ushort nfQuickCheck        : 8;\n"
        out += QByteArray::number( p.nfQuickCheck );
        out += ", ";
//     "        struct {\n"
//     "            ushort special    : 1;\n"
//     "            signed short diff : 15;\n"
//     "        } cases[NumCases];\n"
        out += " { {";
        out += QByteArray::number( p.lowerCaseSpecial );
        out += ", ";
        out += QByteArray::number( p.lowerCaseDiff );
        out += "}, {";
        out += QByteArray::number( p.upperCaseSpecial );
        out += ", ";
        out += QByteArray::number( p.upperCaseDiff );
        out += "}, {";
        out += QByteArray::number( p.titleCaseSpecial );
        out += ", ";
        out += QByteArray::number( p.titleCaseDiff );
        out += "}, {";
        out += QByteArray::number( p.caseFoldSpecial );
        out += ", ";
        out += QByteArray::number( p.caseFoldDiff );
        out += "} }, ";
//     "        ushort graphemeBreakClass  : 5; /* 5 used */\n"
//     "        ushort wordBreakClass      : 5; /* 5 used */\n"
//     "        ushort lineBreakClass      : 6; /* 6 used */\n"
        out += QByteArray::number( p.graphemeBreakClass );
        out += ", ";
        out += QByteArray::number( p.wordBreakClass );
        out += ", ";
        out += QByteArray::number( p.lineBreakClass );
        out += ", ";
//     "        ushort sentenceBreakClass  : 8; /* 4 used */\n"
        out += QByteArray::number( p.sentenceBreakClass );
        out += ", ";
//     "        ushort script              : 8;\n"
        out += QByteArray::number( p.script );
        out += " },";
    }
    if (out.endsWith(','))
        out.chop(1);
    out += "\n};\n\n";


    out += "Q_DECL_CONST_FUNCTION static inline const Properties *qGetProp(uint ucs4) noexcept\n"
           "{\n"
           "    return uc_properties + GET_PROP_INDEX(ucs4);\n"
           "}\n"
           "\n"
           "Q_DECL_CONST_FUNCTION static inline const Properties *qGetProp(ushort ucs2) noexcept\n"
           "{\n"
           "    return uc_properties + GET_PROP_INDEX_UCS2(ucs2);\n"
           "}\n"
           "\n"
           "Q_DECL_CONST_FUNCTION Q_CORE_EXPORT const Properties * QT_FASTCALL properties(uint ucs4) noexcept\n"
           "{\n"
           "    return qGetProp(ucs4);\n"
           "}\n"
           "\n"
           "Q_DECL_CONST_FUNCTION Q_CORE_EXPORT const Properties * QT_FASTCALL properties(ushort ucs2) noexcept\n"
           "{\n"
           "    return qGetProp(ucs2);\n"
           "}\n\n";

    out += "Q_CORE_EXPORT GraphemeBreakClass QT_FASTCALL graphemeBreakClass(uint ucs4) noexcept\n"
           "{\n"
           "    return static_cast<GraphemeBreakClass>(qGetProp(ucs4)->graphemeBreakClass);\n"
           "}\n"
           "\n"
           "Q_CORE_EXPORT WordBreakClass QT_FASTCALL wordBreakClass(uint ucs4) noexcept\n"
           "{\n"
           "    return static_cast<WordBreakClass>(qGetProp(ucs4)->wordBreakClass);\n"
           "}\n"
           "\n"
           "Q_CORE_EXPORT SentenceBreakClass QT_FASTCALL sentenceBreakClass(uint ucs4) noexcept\n"
           "{\n"
           "    return static_cast<SentenceBreakClass>(qGetProp(ucs4)->sentenceBreakClass);\n"
           "}\n"
           "\n"
           "Q_CORE_EXPORT LineBreakClass QT_FASTCALL lineBreakClass(uint ucs4) noexcept\n"
           "{\n"
           "    return static_cast<LineBreakClass>(qGetProp(ucs4)->lineBreakClass);\n"
           "}\n"
           "\n";

    return out;
}

static QByteArray createSpecialCaseMap()
{
    qDebug("createSpecialCaseMap:");

    QByteArray out;

    out += "static const unsigned short specialCaseMap[] = {\n"
           "    0x0, // placeholder";
    int i = 1;
    while (i < specialCaseMap.size()) {
        out += "\n   ";
        int n = specialCaseMap.at(i);
        for (int j = 0; j <= n; ++j) {
            out += QByteArray(" 0x") + QByteArray::number(specialCaseMap.at(i+j), 16);
            out += ",";
        }
        i += n + 1;
    }
    out.chop(1);
    out += "\n};\n\n";

    qDebug("    memory usage: %ld bytes", specialCaseMap.size()*sizeof(unsigned short));

    return out;
}


static QByteArray createCompositionInfo()
{
    qDebug("createCompositionInfo: highestComposedCharacter=0x%x", highestComposedCharacter);

    const int BMP_BLOCKSIZE = 16;
    const int BMP_SHIFT = 4;
    const int BMP_END = 0x3400; // start of Han
    const int SMP_END = 0x30000;
    const int SMP_BLOCKSIZE = 256;
    const int SMP_SHIFT = 8;

    if (SMP_END <= highestComposedCharacter)
        qFatal("end of table smaller than highest composed character 0x%x", highestComposedCharacter);

    QVector<unsigned short> decompositions;
    int tableIndex = 0;

    QList<UniqueBlock> uniqueBlocks;
    QVector<int> blockMap;
    int used = 0;

    for (int block = 0; block < BMP_END/BMP_BLOCKSIZE; ++block) {
        UniqueBlock b;
        b.values.reserve(BMP_BLOCKSIZE);
        for (int i = 0; i < BMP_BLOCKSIZE; ++i) {
            int uc = block*BMP_BLOCKSIZE + i;
            UnicodeData &d = UnicodeData::valueRef(uc);
            if (!d.decomposition.isEmpty()) {
                int utf16Length = 0;
                decompositions.append(0);
                for (int j = 0; j < d.decomposition.size(); ++j) {
                    int code = d.decomposition.at(j);
                    if (QChar::requiresSurrogates(code)) {
                        // save as surrogate pair
                        decompositions.append(QChar::highSurrogate(code));
                        decompositions.append(QChar::lowSurrogate(code));
                        utf16Length += 2;
                    } else {
                        decompositions.append(code);
                        utf16Length++;
                    }
                }
                decompositions[tableIndex] = d.decompositionType + (utf16Length<<8);
                b.values.append(tableIndex);
                tableIndex += utf16Length + 1;
            } else {
                b.values.append(0xffff);
            }
        }
        int index = uniqueBlocks.indexOf(b);
        if (index == -1) {
            index = uniqueBlocks.size();
            b.index = used;
            used += BMP_BLOCKSIZE;
            uniqueBlocks.append(b);
        }
        blockMap.append(uniqueBlocks.at(index).index);
    }
    int bmp_blocks = uniqueBlocks.size();

    for (int block = BMP_END/SMP_BLOCKSIZE; block < SMP_END/SMP_BLOCKSIZE; ++block) {
        UniqueBlock b;
        b.values.reserve(SMP_BLOCKSIZE);
        for (int i = 0; i < SMP_BLOCKSIZE; ++i) {
            int uc = block*SMP_BLOCKSIZE + i;
            UnicodeData &d = UnicodeData::valueRef(uc);
            if (!d.decomposition.isEmpty()) {
                int utf16Length = 0;
                decompositions.append(0);
                for (int j = 0; j < d.decomposition.size(); ++j) {
                    int code = d.decomposition.at(j);
                    if (QChar::requiresSurrogates(code)) {
                        // save as surrogate pair
                        decompositions.append(QChar::highSurrogate(code));
                        decompositions.append(QChar::lowSurrogate(code));
                        utf16Length += 2;
                    } else {
                        decompositions.append(code);
                        utf16Length++;
                    }
                }
                decompositions[tableIndex] = d.decompositionType + (utf16Length<<8);
                b.values.append(tableIndex);
                tableIndex += utf16Length + 1;
            } else {
                b.values.append(0xffff);
            }
        }
        int index = uniqueBlocks.indexOf(b);
        if (index == -1) {
            index = uniqueBlocks.size();
            b.index = used;
            used += SMP_BLOCKSIZE;
            uniqueBlocks.append(b);
        }
        blockMap.append(uniqueBlocks.at(index).index);
    }
    int smp_blocks = uniqueBlocks.size() - bmp_blocks;

    // if the condition below doesn't hold anymore we need to modify our decomposition code
    Q_ASSERT(tableIndex < 0xffff);

    int bmp_block_data = bmp_blocks*BMP_BLOCKSIZE*sizeof(unsigned short);
    int bmp_trie = BMP_END/BMP_BLOCKSIZE*sizeof(unsigned short);
    int bmp_mem = bmp_block_data + bmp_trie;
    qDebug("    %d unique blocks in BMP.", bmp_blocks);
    qDebug("        block data uses: %d bytes", bmp_block_data);
    qDebug("        trie data uses : %d bytes", bmp_trie);

    int smp_block_data = smp_blocks*SMP_BLOCKSIZE*sizeof(unsigned short);
    int smp_trie = (SMP_END-BMP_END)/SMP_BLOCKSIZE*sizeof(unsigned short);
    int smp_mem = smp_block_data + smp_trie;
    qDebug("    %d unique blocks in SMP.", smp_blocks);
    qDebug("        block data uses: %d bytes", smp_block_data);
    qDebug("        trie data uses : %d bytes", smp_trie);

    int decomposition_data = decompositions.size() * 2;
    qDebug("\n        decomposition data uses : %d bytes", decomposition_data);
    qDebug("    memory usage: %d bytes", bmp_mem + smp_mem + decomposition_data);

    Q_ASSERT(blockMap.last() + blockMap.size() < (1<<(sizeof(unsigned short)*8)));

    QByteArray out;

    out += "static const unsigned short uc_decomposition_trie[] = {\n";
    // first write the map
    out += "    // 0 - 0x" + QByteArray::number(BMP_END, 16);
    for (int i = 0; i < BMP_END/BMP_BLOCKSIZE; ++i) {
        if (!(i % 8)) {
            if (out.endsWith(' '))
                out.chop(1);
            if (!((i*BMP_BLOCKSIZE) % 0x1000))
                out += "\n";
            out += "\n    ";
        }
        out += QByteArray::number(blockMap.at(i) + blockMap.size());
        out += ", ";
    }
    if (out.endsWith(' '))
        out.chop(1);
    out += "\n\n    // 0x" + QByteArray::number(BMP_END, 16) + " - 0x" + QByteArray::number(SMP_END, 16) + "\n";
    for (int i = BMP_END/BMP_BLOCKSIZE; i < blockMap.size(); ++i) {
        if (!(i % 8)) {
            if (out.endsWith(' '))
                out.chop(1);
            if (!(i % (0x10000/SMP_BLOCKSIZE)))
                out += "\n";
            out += "\n    ";
        }
        out += QByteArray::number(blockMap.at(i) + blockMap.size());
        out += ", ";
    }
    if (out.endsWith(' '))
        out.chop(1);
    out += "\n";
    // write the data
    for (int i = 0; i < uniqueBlocks.size(); ++i) {
        if (out.endsWith(' '))
            out.chop(1);
        out += "\n";
        const UniqueBlock &b = uniqueBlocks.at(i);
        for (int j = 0; j < b.values.size(); ++j) {
            if (!(j % 8)) {
                if (out.endsWith(' '))
                    out.chop(1);
                out += "\n    ";
            }
            out += "0x" + QByteArray::number(b.values.at(j), 16);
            out += ", ";
        }
    }
    if (out.endsWith(' '))
        out.chop(2);
    out += "\n};\n\n";

    out += "#define GET_DECOMPOSITION_INDEX(ucs4) \\\n"
           "       (ucs4 < 0x" + QByteArray::number(BMP_END, 16) + " \\\n"
           "        ? (uc_decomposition_trie[uc_decomposition_trie[ucs4>>" + QByteArray::number(BMP_SHIFT) +
           "] + (ucs4 & 0x" + QByteArray::number(BMP_BLOCKSIZE-1, 16)+ ")]) \\\n"
           "        : (ucs4 < 0x" + QByteArray::number(SMP_END, 16) + " \\\n"
           "           ? uc_decomposition_trie[uc_decomposition_trie[((ucs4 - 0x" + QByteArray::number(BMP_END, 16) +
           ")>>" + QByteArray::number(SMP_SHIFT) + ") + 0x" + QByteArray::number(BMP_END/BMP_BLOCKSIZE, 16) + "]"
           " + (ucs4 & 0x" + QByteArray::number(SMP_BLOCKSIZE-1, 16) + ")] \\\n"
           "           : 0xffff))\n\n";

    out += "static const unsigned short uc_decomposition_map[] = {";
    for (int i = 0; i < decompositions.size(); ++i) {
        if (!(i % 8)) {
            if (out.endsWith(' '))
                out.chop(1);
            out += "\n    ";
        }
        out += "0x" + QByteArray::number(decompositions.at(i), 16);
        out += ", ";
    }
    if (out.endsWith(' '))
        out.chop(2);
    out += "\n};\n\n";

    return out;
}

static QByteArray createLigatureInfo()
{
    qDebug("createLigatureInfo: numLigatures=%d, highestLigature=0x%x", numLigatures, highestLigature);

    for (int i = 0; i < ligatureHashes.size(); ++i) {
        const QList<Ligature> &l = ligatureHashes.value(i);
        for (int j = 0; j < l.size(); ++j) {
            // if the condition below doesn't hold anymore we need to modify our ligatureHelper code
            Q_ASSERT(QChar::requiresSurrogates(l.at(j).u2) == QChar::requiresSurrogates(l.at(j).ligature) &&
                     QChar::requiresSurrogates(l.at(j).u2) == QChar::requiresSurrogates(l.at(j).u1));
        }
    }

    const int BMP_BLOCKSIZE = 32;
    const int BMP_SHIFT = 5;
    const int BMP_END = 0x3100;
    const int SMP_END = 0x12000;
    const int SMP_BLOCKSIZE = 256;
    const int SMP_SHIFT = 8;

    if (SMP_END <= highestLigature)
        qFatal("end of table smaller than highest ligature character 0x%x", highestLigature);

    QList<unsigned short> ligatures;
    int tableIndex = 0;

    QList<UniqueBlock> uniqueBlocks;
    QVector<int> blockMap;
    int used = 0;

    for (int block = 0; block < BMP_END/BMP_BLOCKSIZE; ++block) {
        UniqueBlock b;
        b.values.reserve(BMP_BLOCKSIZE);
        for (int i = 0; i < BMP_BLOCKSIZE; ++i) {
            int uc = block*BMP_BLOCKSIZE + i;
            QList<Ligature> l = ligatureHashes.value(uc);
            if (!l.isEmpty()) {
                Q_ASSERT(!QChar::requiresSurrogates(uc));
                std::sort(l.begin(), l.end()); // needed for bsearch in ligatureHelper code

                ligatures.append(l.size());
                for (int j = 0; j < l.size(); ++j) {
                    ligatures.append(l.at(j).u1);
                    ligatures.append(l.at(j).ligature);
                }
                b.values.append(tableIndex);
                tableIndex += 2*l.size() + 1;
            } else {
                b.values.append(0xffff);
            }
        }
        int index = uniqueBlocks.indexOf(b);
        if (index == -1) {
            index = uniqueBlocks.size();
            b.index = used;
            used += BMP_BLOCKSIZE;
            uniqueBlocks.append(b);
        }
        blockMap.append(uniqueBlocks.at(index).index);
    }
    int bmp_blocks = uniqueBlocks.size();

    for (int block = BMP_END/SMP_BLOCKSIZE; block < SMP_END/SMP_BLOCKSIZE; ++block) {
        UniqueBlock b;
        b.values.reserve(SMP_BLOCKSIZE);
        for (int i = 0; i < SMP_BLOCKSIZE; ++i) {
            int uc = block*SMP_BLOCKSIZE + i;
            QList<Ligature> l = ligatureHashes.value(uc);
            if (!l.isEmpty()) {
                Q_ASSERT(QChar::requiresSurrogates(uc));
                std::sort(l.begin(), l.end()); // needed for bsearch in ligatureHelper code

                ligatures.append(l.size());
                for (int j = 0; j < l.size(); ++j) {
                    ligatures.append(QChar::highSurrogate(l.at(j).u1));
                    ligatures.append(QChar::lowSurrogate(l.at(j).u1));
                    ligatures.append(QChar::highSurrogate(l.at(j).ligature));
                    ligatures.append(QChar::lowSurrogate(l.at(j).ligature));
                }
                b.values.append(tableIndex);
                tableIndex += 4*l.size() + 1;
            } else {
                b.values.append(0xffff);
            }
        }
        int index = uniqueBlocks.indexOf(b);
        if (index == -1) {
            index = uniqueBlocks.size();
            b.index = used;
            used += SMP_BLOCKSIZE;
            uniqueBlocks.append(b);
        }
        blockMap.append(uniqueBlocks.at(index).index);
    }
    int smp_blocks = uniqueBlocks.size() - bmp_blocks;

    // if the condition below doesn't hold anymore we need to modify our composition code
    Q_ASSERT(tableIndex < 0xffff);

    int bmp_block_data = bmp_blocks*BMP_BLOCKSIZE*sizeof(unsigned short);
    int bmp_trie = BMP_END/BMP_BLOCKSIZE*sizeof(unsigned short);
    int bmp_mem = bmp_block_data + bmp_trie;
    qDebug("    %d unique blocks in BMP.", bmp_blocks);
    qDebug("        block data uses: %d bytes", bmp_block_data);
    qDebug("        trie data uses : %d bytes", bmp_trie);

    int smp_block_data = smp_blocks*SMP_BLOCKSIZE*sizeof(unsigned short);
    int smp_trie = (SMP_END-BMP_END)/SMP_BLOCKSIZE*sizeof(unsigned short);
    int smp_mem = smp_block_data + smp_trie;
    qDebug("    %d unique blocks in SMP.", smp_blocks);
    qDebug("        block data uses: %d bytes", smp_block_data);
    qDebug("        trie data uses : %d bytes", smp_trie);

    int ligature_data = ligatures.size() * 2;
    qDebug("\n        ligature data uses : %d bytes", ligature_data);
    qDebug("    memory usage: %d bytes", bmp_mem + smp_mem + ligature_data);

    Q_ASSERT(blockMap.last() + blockMap.size() < (1<<(sizeof(unsigned short)*8)));

    QByteArray out;

    out += "static const unsigned short uc_ligature_trie[] = {\n";
    // first write the map
    out += "    // 0 - 0x" + QByteArray::number(BMP_END, 16);
    for (int i = 0; i < BMP_END/BMP_BLOCKSIZE; ++i) {
        if (!(i % 8)) {
            if (out.endsWith(' '))
                out.chop(1);
            if (!((i*BMP_BLOCKSIZE) % 0x1000))
                out += "\n";
            out += "\n    ";
        }
        out += QByteArray::number(blockMap.at(i) + blockMap.size());
        out += ", ";
    }
    if (out.endsWith(' '))
        out.chop(1);
    out += "\n\n    // 0x" + QByteArray::number(BMP_END, 16) + " - 0x" + QByteArray::number(SMP_END, 16) + "\n";
    for (int i = BMP_END/BMP_BLOCKSIZE; i < blockMap.size(); ++i) {
        if (!(i % 8)) {
            if (out.endsWith(' '))
                out.chop(1);
            if (!(i % (0x10000/SMP_BLOCKSIZE)))
                out += "\n";
            out += "\n    ";
        }
        out += QByteArray::number(blockMap.at(i) + blockMap.size());
        out += ", ";
    }
    if (out.endsWith(' '))
        out.chop(1);
    out += "\n";
    // write the data
    for (int i = 0; i < uniqueBlocks.size(); ++i) {
        if (out.endsWith(' '))
            out.chop(1);
        out += "\n";
        const UniqueBlock &b = uniqueBlocks.at(i);
        for (int j = 0; j < b.values.size(); ++j) {
            if (!(j % 8)) {
                if (out.endsWith(' '))
                    out.chop(1);
                out += "\n    ";
            }
            out += "0x" + QByteArray::number(b.values.at(j), 16);
            out += ", ";
        }
    }
    if (out.endsWith(' '))
        out.chop(2);
    out += "\n};\n\n";

    out += "#define GET_LIGATURE_INDEX(ucs4) \\\n"
           "       (ucs4 < 0x" + QByteArray::number(BMP_END, 16) + " \\\n"
           "        ? (uc_ligature_trie[uc_ligature_trie[ucs4>>" + QByteArray::number(BMP_SHIFT) +
           "] + (ucs4 & 0x" + QByteArray::number(BMP_BLOCKSIZE-1, 16)+ ")]) \\\n"
           "        : (ucs4 < 0x" + QByteArray::number(SMP_END, 16) + " \\\n"
           "           ? uc_ligature_trie[uc_ligature_trie[((ucs4 - 0x" + QByteArray::number(BMP_END, 16) +
           ")>>" + QByteArray::number(SMP_SHIFT) + ") + 0x" + QByteArray::number(BMP_END/BMP_BLOCKSIZE, 16) + "]"
           " + (ucs4 & 0x" + QByteArray::number(SMP_BLOCKSIZE-1, 16) + ")] \\\n"
           "           : 0xffff))\n\n";

    out += "static const unsigned short uc_ligature_map[] = {";
    for (int i = 0; i < ligatures.size(); ++i) {
        if (!(i % 8)) {
            if (out.endsWith(' '))
                out.chop(1);
            out += "\n    ";
        }
        out += "0x" + QByteArray::number(ligatures.at(i), 16);
        out += ", ";
    }
    if (out.endsWith(' '))
        out.chop(2);
    out += "\n};\n\n";

    return out;
}

QByteArray createCasingInfo()
{
    QByteArray out;

    out += "struct CasingInfo {\n"
           "    uint codePoint : 16;\n"
           "    uint flags : 8;\n"
           "    uint offset : 8;\n"
           "};\n\n";

    return out;
}


int main(int, char **)
{
    initAgeMap();
    initCategoryMap();
    initDecompositionMap();
    initDirectionMap();
    initJoiningMap();
    initGraphemeBreak();
    initWordBreak();
    initSentenceBreak();
    initLineBreak();
    initScriptMap();

    readUnicodeData();
    readBidiMirroring();
    readArabicShaping();
    readDerivedAge();
    readDerivedNormalizationProps();
    readSpecialCasing();
    readCaseFolding();
    // readBlocks();
    readScripts();
    readGraphemeBreak();
    readWordBreak();
    readSentenceBreak();
    readLineBreak();

    computeUniqueProperties();
    QByteArray properties = createPropertyInfo();
    QByteArray specialCases = createSpecialCaseMap();
    QByteArray compositions = createCompositionInfo();
    QByteArray ligatures = createLigatureInfo();
    QByteArray normalizationCorrections = createNormalizationCorrections();

    QByteArray header =
        "/****************************************************************************\n"
        "**\n"
        "** Copyright (C) 2020 The Qt Company Ltd.\n"
        "** Contact: https://www.qt.io/licensing/\n"
        "**\n"
        "** This file is part of the QtCore module of the Qt Toolkit.\n"
        "**\n"
        "** $QT_BEGIN_LICENSE:LGPL$\n"
        "** Commercial License Usage\n"
        "** Licensees holding valid commercial Qt licenses may use this file in\n"
        "** accordance with the commercial license agreement provided with the\n"
        "** Software or, alternatively, in accordance with the terms contained in\n"
        "** a written agreement between you and The Qt Company. For licensing terms\n"
        "** and conditions see https://www.qt.io/terms-conditions. For further\n"
        "** information use the contact form at https://www.qt.io/contact-us.\n"
        "**\n"
        "** GNU Lesser General Public License Usage\n"
        "** Alternatively, this file may be used under the terms of the GNU Lesser\n"
        "** General Public License version 3 as published by the Free Software\n"
        "** Foundation and appearing in the file LICENSE.LGPL3 included in the\n"
        "** packaging of this file. Please review the following information to\n"
        "** ensure the GNU Lesser General Public License version 3 requirements\n"
        "** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.\n"
        "**\n"
        "** GNU General Public License Usage\n"
        "** Alternatively, this file may be used under the terms of the GNU\n"
        "** General Public License version 2.0 or (at your option) the GNU General\n"
        "** Public license version 3 or any later version approved by the KDE Free\n"
        "** Qt Foundation. The licenses are as published by the Free Software\n"
        "** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3\n"
        "** included in the packaging of this file. Please review the following\n"
        "** information to ensure the GNU General Public License requirements will\n"
        "** be met: https://www.gnu.org/licenses/gpl-2.0.html and\n"
        "** https://www.gnu.org/licenses/gpl-3.0.html.\n"
        "**\n"
        "** $QT_END_LICENSE$\n"
        "**\n"
        "****************************************************************************/\n\n";

    QByteArray note =
        "/* This file is autogenerated from the Unicode " DATA_VERSION_S " database. Do not edit */\n\n";

    QByteArray warning =
        "//\n"
        "//  W A R N I N G\n"
        "//  -------------\n"
        "//\n"
        "// This file is not part of the Qt API.  It exists for the convenience\n"
        "// of internal files.  This header file may change from version to version\n"
        "// without notice, or even be removed.\n"
        "//\n"
        "// We mean it.\n"
        "//\n\n";

    QFile f("../../src/corelib/text/qunicodetables.cpp");
    f.open(QFile::WriteOnly|QFile::Truncate);
    f.write(header);
    f.write(note);
    f.write("#include \"qunicodetables_p.h\"\n\n");
    f.write("QT_BEGIN_NAMESPACE\n\n");
    f.write("namespace QUnicodeTables {\n\n");
    f.write(properties);
    f.write("\n");
    f.write(specialCases);
    f.write("\n");
    f.write(compositions);
    f.write(ligatures);
    f.write("\n");
    f.write(normalizationCorrections);
    f.write("} // namespace QUnicodeTables\n\n");
    f.write("using namespace QUnicodeTables;\n\n");
    f.write("QT_END_NAMESPACE\n");
    f.close();

    f.setFileName("../../src/corelib/text/qunicodetables_p.h");
    f.open(QFile::WriteOnly | QFile::Truncate);
    f.write(header);
    f.write(note);
    f.write(warning);
    f.write("#ifndef QUNICODETABLES_P_H\n"
            "#define QUNICODETABLES_P_H\n\n"
            "#include <QtCore/private/qglobal_p.h>\n\n"
            "#include <QtCore/qchar.h>\n\n"
            "QT_BEGIN_NAMESPACE\n\n");
    f.write("#define UNICODE_DATA_VERSION " DATA_VERSION_STR "\n\n");
    f.write("namespace QUnicodeTables {\n\n");
    f.write(property_string);
    f.write(sizeOfPropertiesStructCheck);
    f.write(grapheme_break_class_string);
    f.write(word_break_class_string);
    f.write(sentence_break_class_string);
    f.write(line_break_class_string);
    f.write(methods);
    f.write("} // namespace QUnicodeTables\n\n"
            "QT_END_NAMESPACE\n\n"
            "#endif // QUNICODETABLES_P_H\n");
    f.close();

    qDebug() << "maxMirroredDiff  = " << hex << maxMirroredDiff;
    qDebug() << "maxLowerCaseDiff = " << hex << maxLowerCaseDiff;
    qDebug() << "maxUpperCaseDiff = " << hex << maxUpperCaseDiff;
    qDebug() << "maxTitleCaseDiff = " << hex << maxTitleCaseDiff;
    qDebug() << "maxCaseFoldDiff  = " << hex << maxCaseFoldDiff;
#if 0
//     dump(0, 0x7f);
//     dump(0x620, 0x640);
//     dump(0x10000, 0x10020);
//     dump(0x10800, 0x10820);

    qDebug("decompositionLength used:");
    int totalcompositions = 0;
    int sum = 0;
    for (int i = 1; i < 20; ++i) {
        qDebug("    length %d used %d times", i, decompositionLength.value(i, 0));
        totalcompositions += i*decompositionLength.value(i, 0);
        sum += decompositionLength.value(i, 0);
    }
    qDebug("    len decomposition map %d, average length %f, num composed chars %d",
           totalcompositions, (float)totalcompositions/(float)sum, sum);
    qDebug("highest composed character %x", highestComposedCharacter);
    qDebug("num ligatures = %d highest=%x, maxLength=%d", numLigatures, highestLigature, longestLigature);

    qBubbleSort(ligatures);
    for (int i = 0; i < ligatures.size(); ++i)
        qDebug("%s", ligatures.at(i).data());

//     qDebug("combiningClass usage:");
//     int numClasses = 0;
//     for (int i = 0; i < 255; ++i) {
//         int num = combiningClassUsage.value(i, 0);
//         if (num) {
//             ++numClasses;
//             qDebug("    combiningClass %d used %d times", i, num);
//         }
//     }
//     qDebug("total of %d combining classes used", numClasses);

#endif
}