<commit_before>/**
 * persianParser.js
 *
 * Converts persian text to work with iOS/Android webfonts,
 * parses the text and changes it's unicode values to work
 * on iOS/Anroid browsers.
 *
 * Copyright 2013, <NAME> <<EMAIL>>
 * http://sallar.me
 *
 * Based on the code from:
 * http://ixdc.net/arabic/
 * 
 * Released under the MIT Licence
 * http://opensource.org/licenses/MIT
 *
 * Github: https://github.com/sallar/persianParser
 * Version: 0.1.0
 */
(function (){

    /**
     * Configuration and default variables
     */
    var VERSION   = "0.1.0"
      , hasModule = (typeof module !== 'undefined' && module.exports)
      , chars     = ''
      , special   = "اأإآدذرزوؤء";
        special  += String.fromCharCode(0xFEFC,0xFEFB,0xFEF8,0xFEF7,0xFEFA,0xFEF9,0xFEF6,0xFEF5);

    /**
     * persian characters and their associated alternatives
     *
     * structure :
     * character : [ solo, begin, middle, end, Lam-Alef-1, Lam-Alef-2 ]
     */
        characterAlternatives = {
            "ا": [0xFE8D, 0xFE8D, 0xFE8E, 0xFE8E, 0xFEFC, 0xFEFB],
            "أ": [0xFE83, 0xFE83, 0xFE84, 0xFE84, 0xFEF8, 0xFEF7],
            "إ": [0xFE87, 0xFE87, 0xFE88, 0xFE88, 0xFEFA, 0xFEF9],
            "آ": [0xFE81, 0xFE81, 0xFE82, 0xFE82, 0xFEF6, 0xFEF5],
            "ب": [0xFE8F, 0xFE91, 0xFE92, 0xFE90],
            "پ": [0xFB56, 0xFB58, 0xFB59, 0xFB57],
            "ت": [0xFE95, 0xFE97, 0xFE98, 0xFE96],
            "ث": [0xFE99, 0xFE9B, 0xFE9C, 0xFE9A],
            "ج": [0xFE9D, 0xFE9F, 0xFEA0, 0xFE9E],
            "چ": [0xfb7a, 0xfb7c, 0xfb7d, 0xfb7b],
            "ح": [0xFEA1, 0xFEA3, 0xFEA4, 0xFEA2],
            "خ": [0xFEA5, 0xFEA7, 0xFEA8, 0xFEA6],
            "د": [0xFEA9, 0xFEA9, 0xFEAA, 0xFEAA],
            "ذ": [0xFEAB, 0xFEAB, 0xFEAC, 0xFEAC],
            "ر": [0xFEAD, 0xFEAD, 0xFEAE, 0xFEAE],
            "ز": [0xFEAF, 0xFEAF, 0xFEB0, 0xFEB0],
            "ژ": [0xfb8a, 0xfb8a, 0xfb8b, 0xfb8b],
            "س": [0xFEB1, 0xFEB3, 0xFEB4, 0xFEB2],
            "ش": [0xFEB5, 0xFEB7, 0xFEB8, 0xFEB6],
            "ص": [0xFEB9, 0xFEBB, 0xFEBC, 0xFEBA],
            "ض": [0xFEBD, 0xFEBF, 0xFEC0, 0xFEBE],
            "ط": [0xFEC1, 0xFEC3, 0xFEC4, 0xFEC2],
            "ظ": [0xFEC5, 0xFEC7, 0xFEC8, 0xFEC6],
            "ع": [0xFEC9, 0xFECB, 0xFECC, 0xFECA],
            "غ": [0xFECD, 0xFECF, 0xFED0, 0xFECE],
            "ف": [0xFED1, 0xFED3, 0xFED4, 0xFED2],
            "ق": [0xFED5, 0xFED7, 0xFED8, 0xFED6],
            "ك": [0xFED9, 0xFEDB, 0xFEDC, 0xFEDA],
            "ک": [0xfed9, 0xfedb, 0xfedc, 0xfeda],
            "گ": [0xfb92, 0xfb94, 0xfb95, 0xfb93],
            "ل": [0xFEDD, 0xFEDF, 0xFEE0, 0xFEDE],
            "م": [0xFEE1, 0xFEE3, 0xFEE4, 0xFEE2],
            "ن": [0xFEE5, 0xFEE7, 0xFEE8, 0xFEE6],
            "ه": [0xFEE9, 0xFEEB, 0xFEEC, 0xFEEA],<commit_msg>Update persianParser.js<commit_after>/**
 * persianParser.js
 *
 * Converts persian text to work with iOS/Android webfonts,
 * parses the text and changes it's unicode values to work
 * on iOS/Anroid browsers.
 *
 * Copyright 2013, <NAME> <<EMAIL>>
 * http://sallar.me
 *
 * Based on the code from:
 * http://ixdc.net/arabic/
 * 
 * Released under the MIT Licence
 * http://opensource.org/licenses/MIT
 *
 * Github: https://github.com/sallar/persianParser
 * Version: 0.1.0
 */
(function (){

    /**
     * Configuration and default variables
     */
    var VERSION   = "0.1.0"
      , hasModule = (typeof module !== 'undefined' && module.exports)
      , chars     = ''
      , special   = "اأإآدذرزوؤء";
        special  += String.fromCharCode(0xFEFC,0xFEFB,0xFEF8,0xFEF7,0xFEFA,0xFEF9,0xFEF6,0xFEF5);

    /**
     * persian characters and their associated alternatives
     *
     * structure :
     * character : [ solo, begin, middle, end, Lam-Alef-1, Lam-Alef-2 ]
     */
        characterAlternatives = {
            "ا": [0xFE8D, 0xFE8D, 0xFE8E, 0xFE8E, 0xFEFC, 0xFEFB],
            "أ": [0xFE83, 0xFE83, 0xFE84, 0xFE84, 0xFEF8, 0xFEF7],
            "إ": [0xFE87, 0xFE87, 0xFE88, 0xFE88, 0xFEFA, 0xFEF9],
            "آ": [0xFE81, 0xFE81, 0xFE82, 0xFE82, 0xFEF6, 0xFEF5],
            "ب": [0xFE8F, 0xFE91, 0xFE92, 0xFE90],
            "پ": [0xFB56, 0xFB58, 0xFB59, 0xFB57],
            "ت": [0xFE95, 0xFE97, 0xFE98, 0xFE96],
            "ث": [0xFE99, 0xFE9B, 0xFE9C, 0xFE9A],
            "ج": [0xFE9D, 0xFE9F, 0xFEA0, 0xFE9E],
            "چ": [0xfb7a, 0xfb7c, 0xfb7d, 0xfb7b],
            "ح": [0xFEA1, 0xFEA3, 0xFEA4, 0xFEA2],
            "خ": [0xFEA5, 0xFEA7, 0xFEA8, 0xFEA6],
            "د": [0xFEA9, 0xFEA9, 0xFEAA, 0xFEAA],
            "ذ": [0xFEAB, 0xFEAB, 0xFEAC, 0xFEAC],
            "ر": [0xFEAD, 0xFEAD, 0xFEAE, 0xFEAE],
            "ز": [0xFEAF, 0xFEAF, 0xFEB0, 0xFEB0],
            "ژ": [0xfb8a, 0xfb8a, 0xfb8b, 0xfb8b],
            "س": [0xFEB1, 0xFEB3, 0xFEB4, 0xFEB2],
            "ش": [0xFEB5, 0xFEB7, 0xFEB8, 0xFEB6],
            "ص": [0xFEB9, 0xFEBB, 0xFEBC, 0xFEBA],
            "ض": [0xFEBD, 0xFEBF, 0xFEC0, 0xFEBE],
            "ط": [0xFEC1, 0xFEC3, 0xFEC4, 0xFEC2],
            "ظ": [0xFEC5, 0xFEC7, 0xFEC8, 0xFEC6],
            "ع": [0xFEC9, 0xFECB, 0xFECC, 0xFECA],
            "غ": [0xFECD, 0xFECF, 0xFED0, 0xFECE],
            "ف": [0xFED1, 0xFED3, 0xFED4, 0xFED2],
            "ق": [0xFED5, 0xFED7, 0xFED8, 0xFED6],
            "ك": [0xFED9, 0xFEDB, 0xFEDC, 0xFEDA],
            "ک": [0xFB8E, 0xFB90, 0xFB91, 0xFB8F],
            "گ": [0xfb92, 0xfb94, 0xfb95, 0xfb93],
            "ل": [0xFEDD, 0xFEDF, 0xFEE0, 0xFEDE],
            "م": [0xFEE1, 0xFEE3, 0xFEE4, 0xFEE2],
            "ن": [0xFEE5, 0xFEE7, 0xFEE8, 0xFEE6],
            "ه": [0xFEE9, 0xFEEB, 0xFEEC, 0xFEE9],
{'fraction_non_alphanumeric': 0.1612041325258283, 'fraction_numerical': 0.10865692910580692, 'mean_word_length': 2.2740524781341107, 'pattern_counts': {'":': 68, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6766835', 'n_tokens_mistral': 2762, 'n_tokens_neox': 2664, 'n_words': 510}