natural language toolkit an crubadan ngrams reader c 20012023 nltk project avital pekker avital pekkerutoronto ca url https www nltk org for license information see license txt an nltk interface for the ngram statistics gathered from the corpora for each language using an crubadan there are multiple potential applications for the data but this reader was created with the goal of using it in the context of language identification for details about an crubadan this data and its potential uses see http borel slu educrubadanindex html a corpus reader used to access language an crubadan ngram files return ngram freqdist for a specific language given iso 6393 language code if lang not in self alllangfreq self alllangfreqlang self loadlangngramslang return self alllangfreqlang def langsself return internal crubadan code based on iso 6393 code for i in self langmappingdata if i1 lower lang lower return i0 def crubadantoisoself lang load language mappings between codes and description from table txt if isinstanceself root zipfilepathpointer raise runtimeerror please install the crubadan corpus first use nltk download mapperfile path joinself root self langmapperfile if self langmapperfile not in self fileids raise runtimeerrorcould not find language mapper file mapperfile with openmapperfile encodingutf8 as raw stripraw raw read strip self langmappingdata row splitt for row in stripraw splitn def loadlangngramsself lang natural language toolkit an crubadan n grams reader c 2001 2023 nltk project avital pekker avital pekker utoronto ca url https www nltk org for license information see license txt an nltk interface for the n gram statistics gathered from the corpora for each language using an crubadan there are multiple potential applications for the data but this reader was created with the goal of using it in the context of language identification for details about an crubadan this data and its potential uses see http borel slu edu crubadan index html a corpus reader used to access language an crubadan n gram files return n gram freqdist for a specific language given iso 639 3 language code return a list of supported languages as iso 639 3 codes return internal crubadan code based on iso 639 3 code return iso 639 3 code given internal crubadan code load language mappings between codes and description from table txt load single n gram language file given the iso 639 3 language code and return its freqdist
import re
from os import path

from nltk.corpus.reader import CorpusReader
from nltk.data import ZipFilePathPointer
from nltk.probability import FreqDist


class CrubadanCorpusReader(CorpusReader):
    

    _LANG_MAPPER_FILE = "table.txt"
    _all_lang_freq = {}

    def __init__(self, root, fileids, encoding="utf8", tagset=None):
        super().__init__(root, fileids, encoding="utf8")
        self._lang_mapping_data = []
        self._load_lang_mapping_data()

    def lang_freq(self, lang):
        

        if lang not in self._all_lang_freq:
            self._all_lang_freq[lang] = self._load_lang_ngrams(lang)

        return self._all_lang_freq[lang]

    def langs(self):
        
        return [row[1] for row in self._lang_mapping_data]

    def iso_to_crubadan(self, lang):
        
        for i in self._lang_mapping_data:
            if i[1].lower() == lang.lower():
                return i[0]

    def crubadan_to_iso(self, lang):
        
        for i in self._lang_mapping_data:
            if i[0].lower() == lang.lower():
                return i[1]

    def _load_lang_mapping_data(self):
        
        if isinstance(self.root, ZipFilePathPointer):
            raise RuntimeError(
                "Please install the 'crubadan' corpus first, use nltk.download()"
            )

        mapper_file = path.join(self.root, self._LANG_MAPPER_FILE)
        if self._LANG_MAPPER_FILE not in self.fileids():
            raise RuntimeError("Could not find language mapper file: " + mapper_file)

        with open(mapper_file, encoding="utf-8") as raw:
            strip_raw = raw.read().strip()

            self._lang_mapping_data = [row.split("\t") for row in strip_raw.split("\n")]

    def _load_lang_ngrams(self, lang):
        

        if lang not in self.langs():
            raise RuntimeError("Unsupported language.")

        crubadan_code = self.iso_to_crubadan(lang)
        ngram_file = path.join(self.root, crubadan_code + "-3grams.txt")

        if not path.isfile(ngram_file):
            raise RuntimeError("No N-gram file found for requested language.")

        counts = FreqDist()
        with open(ngram_file, encoding="utf-8") as f:
            for line in f:
                data = line.split(" ")

                ngram = data[1].strip("\n")
                freq = int(data[0])

                counts[ngram] = freq

        return counts