<commit_before>angular.module('han', []);

angular.module('han').controller('MainController', function() {


    //this.chars = '天漢以叫集写直今所過骨平説說雪没沒社飯絆青靑鯖歩步海';
    this.chars = '鯖';
    this.charhistory = [this.chars];
    this.variantsMap = variantsMap;
    this.variants = [];
    this.previouslyPasted = "";
    this.fonts = [
        {
            countryCode: 'TW',
            shortName: 'Kai',
            cssClass: 'taiwan-kai-pc',
            glyphDict: CMEXc1_glyphdict
        },
        {
            countryCode: 'TW',
            shortName: 'Song',
            cssClass: 'taiwan-song-pc',
            glyphDict: CMEXa1_glyphdict
        },
        {
            countryCode: 'JP',
            shortName: 'Mincho',
            cssClass: 'japan-ipa-mincho',
            glyphDict: IPAMincho_glyphdict
        },
        {
            countryCode: 'JP',
            shortName: 'Gothic',
            cssClass: 'japan-ipa-gothic',
            glyphDict: IPAGothic_glyphdict
        },
        {
            countryCode: 'HK',
            shortName: 'Ming',
            cssClass: 'hong-kong-ming',
            glyphDict: Ming_glyphdict
        },
        {
            countryCode: 'CN',
            shortName: 'Sans',
            cssClass: 'source-han-sans-cn',
            glyphDict: SourceHanSansCN_Regular_glyphdict
        },
        {
            countryCode: 'CN',
            shortName: 'Song',
            cssClass: 'ar-pl-baosong',
            glyphDict: ARPLBaosong2GBK_Light_glyphdict
        },
        {
            countryCode: 'KR',
            shortName: 'Gothic',
            cssClass: 'nanum-gothic',
            glyphDict: NanumGothic_glyphdict
        },
        {
            countryCode: 'KR',
            shortName: 'Seoul',
            cssClass: 'seoul-hangang',
            glyphDict: SeoulHangangL_glyphdict
        }
    ];

    var textBox = document.getElementById('TextBox');

    document.documentElement.addEventListener('touchstart', function(event) {
        if (event.target != textBox) {
            textBox.blur();
        } else if (document.activeElement == textBox) {
            cordova.plugins.Keyboard.disableScroll(true);
        }
    });

    document.documentElement.addEventListener('touchend', function(event) {
        if (document.activeElement == textBox) {
            cordova.plugins.Keyboard.disableScroll(false);
        }
    });

    this.toChars = function() {
        this.variants = this.variantsMap[this.chars.charCodeAt(0)];
        var variantChars = [];
        if (this.variants) {
            for (i = 0; i < this.variants.length; i++) {
                variantChars.push(String.fromCharCode(this.variants[i]));
            }
        }
        return variantChars;
    };

    this.appendToHistory = function (chars) {

        trimmed = chars.trim();
        if (trimmed != "" && trimmed.charCodeAt(0) >= 0x3400 && trimmed.charCodeAt(0) <= 0x9FFF) {
            this.charhistory.push(trimmed[0]);
        }

        d = {};
        noduplicate = [];
        for (var i = this.charhistory.length; i--; i >= 0) {

          if (this.charhistory[i] in d) continue;
          noduplicate.unshift(this.charhistory[i]);
          d[this.charhistory[i]] = 1;<commit_msg>Added CJK Compatibility Ideographs to Chinese character check<commit_after>angular.module('han', []);

angular.module('han').controller('MainController', function() {


    //this.chars = '天漢以叫集写直今所過骨平説說雪没沒社飯絆青靑鯖歩步海';
    this.chars = '鯖';
    this.charhistory = [this.chars];
    this.variantsMap = variantsMap;
    this.variants = [];
    this.previouslyPasted = "";
    this.fonts = [
        {
            countryCode: 'TW',
            shortName: 'Kai',
            cssClass: 'taiwan-kai-pc',
            glyphDict: CMEXc1_glyphdict
        },
        {
            countryCode: 'TW',
            shortName: 'Song',
            cssClass: 'taiwan-song-pc',
            glyphDict: CMEXa1_glyphdict
        },
        {
            countryCode: 'JP',
            shortName: 'Mincho',
            cssClass: 'japan-ipa-mincho',
            glyphDict: IPAMincho_glyphdict
        },
        {
            countryCode: 'JP',
            shortName: 'Gothic',
            cssClass: 'japan-ipa-gothic',
            glyphDict: IPAGothic_glyphdict
        },
        {
            countryCode: 'HK',
            shortName: 'Ming',
            cssClass: 'hong-kong-ming',
            glyphDict: Ming_glyphdict
        },
        {
            countryCode: 'CN',
            shortName: 'Sans',
            cssClass: 'source-han-sans-cn',
            glyphDict: SourceHanSansCN_Regular_glyphdict
        },
        {
            countryCode: 'CN',
            shortName: 'Song',
            cssClass: 'ar-pl-baosong',
            glyphDict: ARPLBaosong2GBK_Light_glyphdict
        },
        {
            countryCode: 'KR',
            shortName: 'Gothic',
            cssClass: 'nanum-gothic',
            glyphDict: NanumGothic_glyphdict
        },
        {
            countryCode: 'KR',
            shortName: 'Seoul',
            cssClass: 'seoul-hangang',
            glyphDict: SeoulHangangL_glyphdict
        }
    ];

    var textBox = document.getElementById('TextBox');

    document.documentElement.addEventListener('touchstart', function(event) {
        if (event.target != textBox) {
            textBox.blur();
        } else if (document.activeElement == textBox) {
            cordova.plugins.Keyboard.disableScroll(true);
        }
    });

    document.documentElement.addEventListener('touchend', function(event) {
        if (document.activeElement == textBox) {
            cordova.plugins.Keyboard.disableScroll(false);
        }
    });

    this.toChars = function() {
        this.variants = this.variantsMap[this.chars.charCodeAt(0)];
        var variantChars = [];
        if (this.variants) {
            for (i = 0; i < this.variants.length; i++) {
                variantChars.push(String.fromCharCode(this.variants[i]));
            }
        }
        return variantChars;
    };

    function isChineseCharacter(char) {
        charcode = char.charCodeAt(0);
        if ((charcode >= 0x3400 && charcode <= 0x9FFF) || (charcode >= 0xF900 && charcode <= 0xFAFF)) {
            return true;
        }
        return false;
    }

    this.appendToHistory = function (chars) {

        trimmed = chars.trim();
        if ((trimmed != "") && isChineseCharacter(trimmed[0])) {
            this.charhistory.push(trimmed[0]);
        }

        d = {};
        noduplicate = [];
        for (var i = this.charhistory.length; i--; i >= 0) {

          if (this.charhistory[i] in d) continue;
          noduplicate.unshift(this.charhistory[i]);
          d[this.charhistory[i]] = 1;
{'fraction_non_alphanumeric': 0.1278016921478403, 'fraction_numerical': 0.005788926822027609, 'mean_word_length': 1.726831242411979, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7438342', 'n_tokens_mistral': 2185, 'n_tokens_neox': 2061, 'n_words': 324}