udhr corpus reader it mostly deals with encodings the following files are not fully decodable because they were truncated at wrong bytes unfortunately encodings required for reading the following files are not supported by python the following files are encoded for specific fonts what are these the following files are unintended yeah the following files are not fully decodable because they were truncated at wrong bytes unfortunately encodings required for reading the following files are not supported by python latin3 raises an exception the following files are encoded for specific fonts what are these the following files are unintended
from nltk.corpus.reader.plaintext import PlaintextCorpusReader
from nltk.corpus.reader.util import find_corpus_fileids


class UdhrCorpusReader(PlaintextCorpusReader):

    ENCODINGS = [
        (".*-Latin1$", "latin-1"),
        (".*-Hebrew$", "hebrew"),
        (".*-Arabic$", "cp1256"),
        ("Czech_Cesky-UTF8", "cp1250"),  
        ("Polish-Latin2", "cp1250"),
        ("Polish_Polski-Latin2", "cp1250"),
        (".*-Cyrillic$", "cyrillic"),
        (".*-SJIS$", "SJIS"),
        (".*-GB2312$", "GB2312"),
        (".*-Latin2$", "ISO-8859-2"),
        (".*-Greek$", "greek"),
        (".*-UTF8$", "utf-8"),
        ("Hungarian_Magyar-Unicode", "utf-16-le"),
        ("Amahuaca", "latin1"),
        ("Turkish_Turkce-Turkish", "latin5"),
        ("Lithuanian_Lietuviskai-Baltic", "latin4"),
        ("Japanese_Nihongo-EUC", "EUC-JP"),
        ("Japanese_Nihongo-JIS", "iso2022_jp"),
        ("Chinese_Mandarin-HZ", "hz"),
        (r"Abkhaz\-Cyrillic\+Abkh", "cp1251"),
    ]

    SKIP = {
        
        
        "Burmese_Myanmar-UTF8",
        "Japanese_Nihongo-JIS",
        "Chinese_Mandarin-HZ",
        "Chinese_Mandarin-UTF8",
        "Gujarati-UTF8",
        "Hungarian_Magyar-Unicode",
        "Lao-UTF8",
        "Magahi-UTF8",
        "Marathi-UTF8",
        "Tamil-UTF8",
        
        
        "Vietnamese-VPS",
        "Vietnamese-VIQR",
        "Vietnamese-TCVN",
        "Magahi-Agra",
        "Bhojpuri-Agra",
        "Esperanto-T61",  
        
        "Burmese_Myanmar-WinResearcher",
        "Armenian-DallakHelv",
        "Tigrinya_Tigrigna-VG2Main",
        "Amharic-Afenegus6..60375",  
        "Navaho_Dine-Navajo-Navaho-font",
        
        "Azeri_Azerbaijani_Cyrillic-Az.Times.Cyr.Normal0117",
        "Azeri_Azerbaijani_Latin-Az.Times.Lat0117",
        
        "Czech-Latin2-err",
        "Russian_Russky-UTF8~",
    }

    def __init__(self, root="udhr"):
        fileids = find_corpus_fileids(root, r"(?!README|\.).*")
        super().__init__(
            root,
            [fileid for fileid in fileids if fileid not in self.SKIP],
            encoding=self.ENCODINGS,
        )