natural language toolkit word list corpus reader c 20012023 nltk project steven bird stevenbird1gmail com edward loper edlopergmail com url https www nltk org for license information see license txt this is a class to read the panlex swadesh list from david kamholz jonathan pool and susan m colowick 2014 panlex building a resource for panlingual lexical translation in lrec http www lrecconf orgproceedingslrec2014pdf1029paper pdf license cc0 1 0 universal https creativecommons orgpublicdomainzero1 0legalcode find the swadesh size using the fileids path return a list of liststr return a list of liststr return a tuple of words for the specified fileids natural language toolkit word list corpus reader c 2001 2023 nltk project steven bird stevenbird1 gmail com edward loper edloper gmail com url https www nltk org for license information see license txt 1 panlex uid 2 iso 639 language code 3 iso 639 language type see readme 4 normal scripts of expressions 5 panlex default name 6 uid of the language variety in which the default name is an expression this is a class to read the panlex swadesh list from david kamholz jonathan pool and susan m colowick 2014 panlex building a resource for panlingual lexical translation in lrec http www lrec conf org proceedings lrec2014 pdf 1029_paper pdf license cc0 1 0 universal https creativecommons org publicdomain zero 1 0 legalcode find the swadesh size using the fileids path skip empty lines return a list of list str return a list of list str return a tuple of words for the specified fileids
import re
from collections import defaultdict, namedtuple

from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.corpus.reader.wordlist import WordListCorpusReader
from nltk.tokenize import line_tokenize

PanlexLanguage = namedtuple(
    "PanlexLanguage",
    [
        "panlex_uid",  
        "iso639",  
        "iso639_type",  
        "script",  
        "name",  
        "langvar_uid",  
    ],
)


class PanlexSwadeshCorpusReader(WordListCorpusReader):
    

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        
        self.swadesh_size = re.match(r"swadesh([0-9].*)\/", self.fileids()[0]).group(1)
        self._languages = {lang.panlex_uid: lang for lang in self.get_languages()}
        self._macro_langauges = self.get_macrolanguages()

    def license(self):
        return "CC0 1.0 Universal"

    def language_codes(self):
        return self._languages.keys()

    def get_languages(self):
        for line in self.raw(f"langs{self.swadesh_size}.txt").split("\n"):
            if not line.strip():  
                continue
            yield PanlexLanguage(*line.strip().split("\t"))

    def get_macrolanguages(self):
        macro_langauges = defaultdict(list)
        for lang in self._languages.values():
            macro_langauges[lang.iso639].append(lang.panlex_uid)
        return macro_langauges

    def words_by_lang(self, lang_code):
        
        fileid = f"swadesh{self.swadesh_size}/{lang_code}.txt"
        return [concept.split("\t") for concept in self.words(fileid)]

    def words_by_iso639(self, iso63_code):
        
        fileids = [
            f"swadesh{self.swadesh_size}/{lang_code}.txt"
            for lang_code in self._macro_langauges[iso63_code]
        ]
        return [
            concept.split("\t") for fileid in fileids for concept in self.words(fileid)
        ]

    def entries(self, fileids=None):
        
        if not fileids:
            fileids = self.fileids()

        wordlists = [self.words(f) for f in fileids]
        return list(zip(*wordlists))