<commit_before>#############################################################################
## data source definition
#############################################################################

<?php
$sourcePath = "/media/disk2/index_dev";

$languages = array(
    'ara' => 'Arabic',
    'eng' => 'English',
    'jpn' => 'Japanese',
    'fra' => 'French',
    'deu' => 'German',
    'spa' => 'Spanish',
    'ita' => 'Italian',
    'vie' => 'Vietnamese',
    'rus' => 'Russian',
    'cmn' => 'Chinese',
    'kor' => 'Korean',
    'nld' => 'Dutch',
    'heb' => 'Hebrew',
    'ind' => 'Indonesian',
    'por' => 'Portuguese',
    'fin' => 'Finnish',
    'bul' => 'Bulgarian',
    'ukr' => 'Ukrainian',
    'ces' => 'Czech',
    'epo' => 'Esperanto',
    'ell' => 'Modern Greek',
    'tur' => 'Turkish',
    'swe' => 'Swedish',
    'nob' => 'Norwegian (Bokmål)',
    'zsm' => 'Malay',
    'est' => 'Estonian',
    'kat' => 'Georgian',
    'pol' => 'Polish', 
    'swh' => 'Swahili', 
    'lat' => 'Latin', 
    'wuu' => 'Shanghainese',
    'arz' => 'Egyptian Arabic',
    'bel' => 'Belarusian',
    'hun' => 'Hungarian',
    'isl' => 'Icelandic',
    'sqi' => 'Albanian',
    'yue' => 'Cantonese',
    'afr' => 'Afrikaans',
    'fao' => 'Faroese',
    'fry' => 'Frisian',
    
    'bre' => 'Breton',
    'ron' => 'Romanian',
    'uig' => 'Uyghur',
    'uzb' => 'Uzbek',
    'non' => 'Norwegian (Nynorsk)',
    'srp' => 'Serbian',
    'tat' => 'Tartar',
    'yid' => 'Yiddish',
    
    'pes' => 'Persian',
    'nan' => 'Teochew',
    'eus' => 'Basque',
    'slk' => 'Slovak',
    'dan' => 'Danish',
    'hye' => 'Armenian',
    'acm' => 'Iraqi Arabic',
    'san' => 'Sanskrit',
    'urd' => 'Urdu',
    'hin' => 'Hindi',
    'ben' => 'Bengali',
    'cycl' => 'CycL',
//    'cat' => 'Catalan',
//    'kaz' => 'Kazakh',
//    'lvs' => 'Latvian',
);

$languageWithStemmer = array(
    "deu"=>0,
    "spa"=>0,
    "fra"=>0,
    "nld"=>0,
    "por"=>0,
    "rus"=>0,
    "fin"=>0,
    "ita"=>0,
    "tur"=>0,
    "swe"=>0,
    "eng"=>0,
); 

    
    
    
$cjkLanguages = array(
    "kor" => 0,
    "cmn" => 0,
    "wuu" => 0,
    "jpn" => 0,
); 


?>
<commit_msg>added cantonese in the list of asian language (as they need special indexing by sphinx)<commit_after>#############################################################################
## data source definition
#############################################################################

<?php
$sourcePath = "/media/disk2/index_dev";

$languages = array(
    'ara' => 'Arabic',
    'eng' => 'English',
    'jpn' => 'Japanese',
    'fra' => 'French',
    'deu' => 'German',
    'spa' => 'Spanish',
    'ita' => 'Italian',
    'vie' => 'Vietnamese',
    'rus' => 'Russian',
    'cmn' => 'Chinese',
    'kor' => 'Korean',
    'nld' => 'Dutch',
    'heb' => 'Hebrew',
    'ind' => 'Indonesian',
    'por' => 'Portuguese',
    'fin' => 'Finnish',
    'bul' => 'Bulgarian',
    'ukr' => 'Ukrainian',
    'ces' => 'Czech',
    'epo' => 'Esperanto',
    'ell' => 'Modern Greek',
    'tur' => 'Turkish',
    'swe' => 'Swedish',
    'nob' => 'Norwegian (Bokmål)',
    'zsm' => 'Malay',
    'est' => 'Estonian',
    'kat' => 'Georgian',
    'pol' => 'Polish', 
    'swh' => 'Swahili', 
    'lat' => 'Latin', 
    'wuu' => 'Shanghainese',
    'arz' => 'Egyptian Arabic',
    'bel' => 'Belarusian',
    'hun' => 'Hungarian',
    'isl' => 'Icelandic',
    'sqi' => 'Albanian',
    'yue' => 'Cantonese',
    'afr' => 'Afrikaans',
    'fao' => 'Faroese',
    'fry' => 'Frisian',
    
    'bre' => 'Breton',
    'ron' => 'Romanian',
    'uig' => 'Uyghur',
    'uzb' => 'Uzbek',
    'non' => 'Norwegian (Nynorsk)',
    'srp' => 'Serbian',
    'tat' => 'Tartar',
    'yid' => 'Yiddish',
    
    'pes' => 'Persian',
    'nan' => 'Teochew',
    'eus' => 'Basque',
    'slk' => 'Slovak',
    'dan' => 'Danish',
    'hye' => 'Armenian',
    'acm' => 'Iraqi Arabic',
    'san' => 'Sanskrit',
    'urd' => 'Urdu',
    'hin' => 'Hindi',
    'ben' => 'Bengali',
    'cycl' => 'CycL',
//    'cat' => 'Catalan',
//    'kaz' => 'Kazakh',
//    'lvs' => 'Latvian',
);

$languageWithStemmer = array(
    "deu"=>0,
    "spa"=>0,
    "fra"=>0,
    "nld"=>0,
    "por"=>0,
    "rus"=>0,
    "fin"=>0,
    "ita"=>0,
    "tur"=>0,
    "swe"=>0,
    "eng"=>0,
); 

    
    
    
$cjkLanguages = array(
    "kor" => 0,
    "cmn" => 0,
    "wuu" => 0,
    "jpn" => 0,
    "yue" => 0,
); 


?>

{'fraction_non_alphanumeric': 0.3243427017225748, 'fraction_numerical': 0.007479601087941976, 'mean_word_length': 2.6291118421052633, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 162, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '640094', 'n_tokens_mistral': 1975, 'n_tokens_neox': 1838, 'n_words': 342}