<commit_before><commit_msg>Create iso_11940.py<commit_after># -*- coding: utf-8 -*-
"""
Transliterating Thai text with ISO 11940

https://en.wikipedia.org/wiki/ISO_11940
"""
_consonants = {
    "ก":"k",
    "ข":"k̄h",
    "ฃ":"ḳ̄h",
    "ค":"kh",
    "ฅ":"k̛h",
    "ฆ":"ḳh",
    "ง":"ng",
    "จ":"c",
    "ฉ":"c̄h",
    "ช":"ch",
    "ซ":"s",
    "ฌ":"c̣h",
    "ญ":"ỵ",
    "ฎ":"ḍ",
    "ฏ":"ṭ",
    "ฐ":"ṭ̄h",
    "ฑ":"ṯh",
    "ฒ":"t̛h",
    "ณ":"ṇ",
    "ด":"d",
    "ต":"t",
    "ถ":"t̄h",
    "ท":"th",
    "ธ":"ṭh",
    "น":"n",
    "บ":"b",
    "ป":"p",
    "ผ":"p̄h",
    "ฝ":"f̄",
    "พ":"ph",
    "ฟ":"f",
    "ภ":"p̣h",
    "ม":"m",
    "ย":"y",
    "ร":"r",
    "ฤ":"v",
    "ล":"l",
    "ฦ":"ł",
    "ว":"w",
    "ศ":"ṣ̄",
    "ษ":"s̛̄",
    "ส":"s̄",
    "ห":"h̄",
    "ฬ":"ḷ",
    "อ":"x",
    "ฮ":"ḥ"  
}

_vowels={
    "ะ":"a",
    "ั":"ạ",
    "า":"ā",
    "ำ":"å",
    "ิ":"i",
    "ี":"ī",
    "ึ":"ụ",
    "ื":"ụ̄",
    "ุ":"u",
    "ู":"ū",
    "เ":"e",
    "แ":"æ",
    "โ":"o",
    "ใ":"ı",
    "ไ":"ị",
    "ฤ":"v",
    "ฤๅ":"vɨ",
    "ฦ":"ł",
    "ฦๅ":"łɨ",
    "ย":"y",
    "ว":"w",
    "อ":"x",
}

_tone_marks = {
    "่":"–̀".replace("–",""),
    "้":"–̂".replace("–",""),
    "๊":"–́".replace("–",""),
    "๋":"–̌".replace("–",""),
    "็":"–̆".replace("–",""),
    "์":"–̒".replace("–",""),
    "–๎".replace("–",""):"~",
    "–ํ".replace("–",""):"–̊".replace("–",""),
    "–ฺ".replace("–",""):"–̥".replace("–",""),
}

_punctuation_and_digits = {
    "ๆ":"«",
    "ฯ":"ǂ",
    "๏":"§",
    "ฯ":"ǀ",
    "๚":"ǁ",
    "๛":"»",
    "๐":"0",
    "๑":"1",
    "๒":"2",
    "๓":"3",
    "๔":"4",
    "๕":"5",
    "๖":"6",
    "๗":"7",
    "๘":"8",
    "๙":"9"
}

_all_dict = {**_consonants, **_vowels, **_tone_marks, **_punctuation_and_digits}
_list_k = _all_dict.keys()


def transliterate(word):
    _new = ""
    for i in word:
        if i in _list_k:
            _new+=_all_dict[i]
        else:
            _new+=i
    return _new
{'fraction_non_alphanumeric': 0.40122199592668023, 'fraction_numerical': 0.018329938900203666, 'mean_word_length': 2.242574257425743, 'pattern_counts': {'":': 90, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4609114', 'n_tokens_mistral': 1070, 'n_tokens_neox': 1074, 'n_words': 133}