src/chardistribution.js

/*
 * The Original Code is Mozilla Universal charset detector code.
 *
 * The Initial Developer of the Original Code is
 * Netscape Communications Corporation.
 * Portions created by the Initial Developer are Copyright (C) 2001
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s):
 *   António Afonso (antonio.afonso gmail.com) - port to JavaScript
 *   Mark Pilgrim - port to Python
 *   Shy Shalom - original C code
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301  USA
 */

var jisfreq = require('./jisfreq');
var euctwfreq = require('./euctwfreq');
var euckrfreq = require('./euckrfreq');
var gb2312freq = require('./gb2312freq');
var big5freq = require('./big5freq');

function CharDistributionAnalysis() {
    var ENOUGH_DATA_THRESHOLD = 1024;
    var SURE_YES = 0.99;
    var SURE_NO = 0.01;
    var MINIMUM_DATA_THRESHOLD = 3;

    var self = this;

    function init() {
        self._mCharToFreqOrder = null; // Mapping table to get frequency order from char order (get from GetOrder())
        self._mTableSize = null; // Size of above table
        self._mTypicalDistributionRatio = null; // This is a constant value which varies from language to language, used in calculating confidence.  See http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html for further detail.
        self.reset();
    }

    /**
     * reset analyser, clear any state
     */
    this.reset = function() {
        this._mDone = false; // If this flag is set to constants.True, detection is done and conclusion has been made
        this._mTotalChars = 0; // Total characters encountered
        this._mFreqChars = 0; // The number of characters whose frequency order is less than 512
    }

    /**
     * feed a character with known length
     */
    this.feed = function(aStr, aCharLen) {
        if( aCharLen == 2 ) {
            // we only care about 2-bytes character in our distribution analysis
            var order = this.getOrder(aStr);
        } else {
            order = -1;
        }
        if( order >= 0 ) {
            this._mTotalChars++;
            // order is valid
            if( order < this._mTableSize ) {
                if( 512 > this._mCharToFreqOrder[order] ) {
                    this._mFreqChars++;
                }
            }
        }
    }

    /**
     * return confidence based on existing data
     */
    this.getConfidence = function() {
        // if we didn't receive any character in our consideration range, return negative answer
        if( this._mTotalChars <= 0 || this._mFreqChars <= MINIMUM_DATA_THRESHOLD) {
            return SURE_NO;
        }
        if( this._mTotalChars != this._mFreqChars ) {
            var r = this._mFreqChars / ((this._mTotalChars - this._mFreqChars) * this._mTypicalDistributionRatio);
            if( r < SURE_YES ) {
                return r;
            }
        }

        // normalize confidence (we don't want to be 100% sure)
        return SURE_YES;
    }

    this.gotEnoughData = function() {
        // It is not necessary to receive all data to draw conclusion. For charset detection,
        // certain amount of data is enough
        return this._mTotalChars > ENOUGH_DATA_THRESHOLD;
    }

    this.getOrder = function(aStr) {
        // We do not handle characters based on the original encoding string, but
        // convert this encoding string to a number, here called order.
        // This allows multiple encodings of a language to share one frequency table.
        return -1;
    }

    init();
}

exports.CharDistributionAnalysis = CharDistributionAnalysis

function EUCTWDistributionAnalysis() {
    CharDistributionAnalysis.apply(this);

    var self = this;

    function init() {
        self._mCharToFreqOrder = euctwfreq.EUCTWCharToFreqOrder;
        self._mTableSize = euctwfreq.EUCTW_TABLE_SIZE;
        self._mTypicalDistributionRatio = euctwfreq.EUCTW_TYPICAL_DISTRIBUTION_RATIO;
    }

    this.getOrder = function(aStr) {
        // for euc-TW encoding, we are interested
        //   first  byte range: 0xc4 -- 0xfe
        //   second byte range: 0xa1 -- 0xfe
        // no validation needed here. State machine has done that
        if( aStr.charCodeAt(0) >= 0xC4 ) {
            return 94 * (aStr.charCodeAt(0) - 0xC4) + aStr.charCodeAt(1) - 0xA1;
        } else {
            return -1;
        }
    }

    init();
}
EUCTWDistributionAnalysis.prototype = new CharDistributionAnalysis();

exports.EUCTWDistributionAnalysis = EUCTWDistributionAnalysis

function EUCKRDistributionAnalysis() {
    CharDistributionAnalysis.apply(this);

    var self = this;

    function init() {
        self._mCharToFreqOrder = euckrfreq.EUCKRCharToFreqOrder;
        self._mTableSize = euckrfreq.EUCKR_TABLE_SIZE;
        self._mTypicalDistributionRatio = euckrfreq.EUCKR_TYPICAL_DISTRIBUTION_RATIO;
    }

    this.getOrder = function(aStr) {
        // for euc-KR encoding, we are interested
        //   first  byte range: 0xb0 -- 0xfe
        //   second byte range: 0xa1 -- 0xfe
        // no validation needed here. State machine has done that
        if( aStr.charCodeAt(0) >= 0xB0 ) {
            return 94 * (aStr.charCodeAt(0) - 0xB0) + aStr.charCodeAt(1) - 0xA1;
        } else {
            return -1;
        }
    }

    init();
}
EUCKRDistributionAnalysis.prototype = new CharDistributionAnalysis();

exports.EUCKRDistributionAnalysis = EUCKRDistributionAnalysis

function GB2312DistributionAnalysis() {
    CharDistributionAnalysis.apply(this);

    var self = this;

    function init() {
        self._mCharToFreqOrder = gb2312freq.GB2312CharToFreqOrder;
        self._mTableSize = gb2312freq.GB2312_TABLE_SIZE;
        self._mTypicalDistributionRatio = gb2312freq.GB2312_TYPICAL_DISTRIBUTION_RATIO;
    }

    this.getOrder = function(aStr) {
        // for GB2312 encoding, we are interested
        //  first  byte range: 0xb0 -- 0xfe
        //  second byte range: 0xa1 -- 0xfe
        // no validation needed here. State machine has done that
        if( aStr.charCodeAt(0) >= 0xB0 && aStr.charCodeAt(1) >= 0xA1 ) {
            return 94 * (aStr.charCodeAt(0) - 0xB0) + aStr.charCodeAt(1) - 0xA1;
        } else {
            return -1;
        }
    }

    init();
}
GB2312DistributionAnalysis.prototype = new CharDistributionAnalysis();

exports.GB2312DistributionAnalysis = GB2312DistributionAnalysis

function Big5DistributionAnalysis() {
    CharDistributionAnalysis.apply(this);

    var self = this;

    function init() {
        self._mCharToFreqOrder = big5freq.Big5CharToFreqOrder;
        self._mTableSize = big5freq.BIG5_TABLE_SIZE;
        self._mTypicalDistributionRatio = big5freq.BIG5_TYPICAL_DISTRIBUTION_RATIO;
    }

    this.getOrder = function(aStr) {
        // for big5 encoding, we are interested
        //   first  byte range: 0xa4 -- 0xfe
        //   second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
        // no validation needed here. State machine has done that
        if( aStr.charCodeAt(0) >= 0xA4 ) {
            if( aStr.charCodeAt(1) >= 0xA1 ) {
                return 157 * (aStr.charCodeAt(0) - 0xA4) + aStr.charCodeAt(1) - 0xA1 + 63;
            } else {
                return 157 * (aStr.charCodeAt(0) - 0xA4) + aStr.charCodeAt(1) - 0x40;
            }
        } else {
            return -1;
        }
    }

    init();
}
Big5DistributionAnalysis.prototype = new CharDistributionAnalysis();

exports.Big5DistributionAnalysis = Big5DistributionAnalysis

function SJISDistributionAnalysis() {
    CharDistributionAnalysis.apply(this);

    var self = this;

    function init() {
        self._mCharToFreqOrder = jisfreq.JISCharToFreqOrder;
        self._mTableSize = jisfreq.JIS_TABLE_SIZE;
        self._mTypicalDistributionRatio = jisfreq.JIS_TYPICAL_DISTRIBUTION_RATIO;
    }

    this.getOrder = function(aStr) {
        // for sjis encoding, we are interested
        //   first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xef
        //   second byte range: 0x40 -- 0x7e,  0x80 -- 0xfc
        // no validation needed here. State machine has done that
        if( aStr.charCodeAt(0) >= 0x81 && aStr.charCodeAt(0) <= 0x9F ) {
            var order = 188 * (aStr.charCodeAt(0) - 0x81);
        } else if( aStr.charCodeAt(0) >= 0xE0 && aStr.charCodeAt(0) <= 0xEF ) {
            order = 188 * (aStr.charCodeAt(0) - 0xE0 + 31);
        } else {
            return -1;
        }
        order += aStr.charCodeAt(1) - 0x40;
        if( aStr.charCodeAt(1) < 0x40 || aStr.charCodeAt(1) === 0x7F || aStr.charCodeAt(1) > 0xFC) {
            order = -1;
        }
        return order;
    }

    init();
}
SJISDistributionAnalysis.prototype = new CharDistributionAnalysis();

exports.SJISDistributionAnalysis = SJISDistributionAnalysis

function EUCJPDistributionAnalysis() {
    CharDistributionAnalysis.apply(this);

    var self = this;

    function init() {
        self._mCharToFreqOrder = jisfreq.JISCharToFreqOrder;
        self._mTableSize = jisfreq.JIS_TABLE_SIZE;
        self._mTypicalDistributionRatio = jisfreq.JIS_TYPICAL_DISTRIBUTION_RATIO;
    }

    this.getOrder = function(aStr) {
        // for euc-JP encoding, we are interested
        //   first  byte range: 0xa0 -- 0xfe
        //   second byte range: 0xa1 -- 0xfe
        // no validation needed here. State machine has done that
        if( aStr[0] >= "\xA0" ) {
            return 94 * (aStr.charCodeAt(0) - 0xA1) + aStr.charCodeAt(1) - 0xA1;
        } else {
            return -1;
        }
    }

    init();
}
EUCJPDistributionAnalysis.prototype = new CharDistributionAnalysis();

exports.EUCJPDistributionAnalysis = EUCJPDistributionAnalysis