natural language toolkit bcp47 language tags c 20222023 nltk project eric kafe kafe ericgmail com url https www nltk org for license information see license txt parse bcp47 composite language tags supports all the main subtags and the usd extension from nltk corpus import bcp47 bcp47 name ocgasconusdfr64 occitan post 1500 gascon pyrnesatlantiques can load a conversion table to wikidata qcodes bcp47 loadwikiq bcp47 wikiq engispanglis q79388 read the bcp47 database super initroot fileids self langcode with self openianalanguagesubtagregistry txt as fp self db self datadictfp read splitn with self opencldrcommonsubdivisionsen xml as fp self subdiv self subdivdict et parsefp iterfindlocaledisplaynamessubdivisionssubdivision self morphology def loadwikiqself convert wikidata list of qcodes to a bcp47 dictionary return pair1 pair0 split1 for pair in line strip splitt for line in lines def subdivdictself subdivs convert the bcp47 language subtag registry to a dictionary self version records0 replacefiledate strip dic dicdeprecated for label in language extlang script region variant redundant grandfathered dicdeprecatedlabel for record in records1 fields field split for field in record strip splitn typ fields01 tag fields11 if typ not in dic dictyp subfields for field in fields2 if lenfield 2 key val field if key not in subfields subfieldskey val else multiple value subfieldskey appendval else multiline field subfieldskey1 field0 strip if deprecated not in record and typ language and key description self langcodesubfieldskey1 tag for key in subfields if lensubfieldskey 1 single value subfieldskey subfieldskey0 if deprecated in record dicdeprecatedtyptag subfields else dictyptag subfields return dic def val2strself val val joinval concatenate all values concatenate subtag values name flgrecord language for label in extlang script region variant extension if label in lgrecord name f lgrecordlabel return name def parsetagself tag convert a bcp47 tag to a colonseparated string of subtag names from nltk corpus import bcp47 bcp47 name calatnesvalencia catalan latin spain valencian natural language toolkit bcp 47 language tags c 2022 2023 nltk project eric kafe kafe eric gmail com url https www nltk org for license information see license txt parse bcp 47 composite language tags supports all the main subtags and the u sd extension from nltk corpus import bcp47 bcp47 name oc gascon u sd fr64 occitan post 1500 gascon pyrénées atlantiques can load a conversion table to wikidata q codes bcp47 load_wiki_q bcp47 wiki_q en gi spanglis q79388 read the bcp 47 database load conversion table to wikidata q codes only if needed convert wikidata list of q codes to a bcp 47 dictionary convert the cldr subdivisions list to a dictionary convert the bcp 47 language subtag registry to a dictionary multiple value multiline field single value return only first value val join val concatenate all values concatenate subtag values convert a bcp 47 tag to a dictionary of labelled subtags cldr regional subdivisions other extension subtags are not supported yet convert a bcp 47 tag to a colon separated string of subtag names from nltk corpus import bcp47 bcp47 name ca latn es valencia catalan latin spain valencian
import re
from warnings import warn
from xml.etree import ElementTree as et

from nltk.corpus.reader import CorpusReader


class BCP47CorpusReader(CorpusReader):
    

    def __init__(self, root, fileids):
        
        super().__init__(root, fileids)
        self.langcode = {}
        with self.open("iana/language-subtag-registry.txt") as fp:
            self.db = self.data_dict(fp.read().split("%%\n"))
        with self.open("cldr/common-subdivisions-en.xml") as fp:
            self.subdiv = self.subdiv_dict(
                et.parse(fp).iterfind("localeDisplayNames/subdivisions/subdivision")
            )
        self.morphology()

    def load_wiki_q(self):
        
        with self.open("cldr/tools-cldr-rdf-external-entityToCode.tsv") as fp:
            self.wiki_q = self.wiki_dict(fp.read().strip().split("\n")[1:])

    def wiki_dict(self, lines):
        
        return {
            pair[1]: pair[0].split("/")[-1]
            for pair in [line.strip().split("\t") for line in lines]
        }

    def subdiv_dict(self, subdivs):
        
        return {sub.attrib["type"]: sub.text for sub in subdivs}

    def morphology(self):
        self.casing = {
            "language": str.lower,
            "extlang": str.lower,
            "script": str.title,
            "region": str.upper,
            "variant": str.lower,
        }
        dig = "[0-9]"
        low = "[a-z]"
        up = "[A-Z]"
        alnum = "[a-zA-Z0-9]"
        self.format = {
            "language": re.compile(f"{low*3}?"),
            "extlang": re.compile(f"{low*3}"),
            "script": re.compile(f"{up}{low*3}"),
            "region": re.compile(f"({up*2})|({dig*3})"),
            "variant": re.compile(f"{alnum*4}{(alnum+'?')*4}"),
            "singleton": re.compile(f"{low}"),
        }

    def data_dict(self, records):
        
        self.version = records[0].replace("File-Date:", "").strip()
        dic = {}
        dic["deprecated"] = {}
        for label in [
            "language",
            "extlang",
            "script",
            "region",
            "variant",
            "redundant",
            "grandfathered",
        ]:
            dic["deprecated"][label] = {}
        for record in records[1:]:
            fields = [field.split(": ") for field in record.strip().split("\n")]
            typ = fields[0][1]
            tag = fields[1][1]
            if typ not in dic:
                dic[typ] = {}
            subfields = {}
            for field in fields[2:]:
                if len(field) == 2:
                    [key, val] = field
                    if key not in subfields:
                        subfields[key] = [val]
                    else:  
                        subfields[key].append(val)
                else:  
                    subfields[key][-1] += " " + field[0].strip()
                if (
                    "Deprecated" not in record
                    and typ == "language"
                    and key == "Description"
                ):
                    self.langcode[subfields[key][-1]] = tag
            for key in subfields:
                if len(subfields[key]) == 1:  
                    subfields[key] = subfields[key][0]
            if "Deprecated" in record:
                dic["deprecated"][typ][tag] = subfields
            else:
                dic[typ][tag] = subfields
        return dic

    def val2str(self, val):
        
        if type(val) == list:
            
            val = val[0]
        return val

    def lang2str(self, lg_record):
        
        name = f"{lg_record['language']}"
        for label in ["extlang", "script", "region", "variant", "extension"]:
            if label in lg_record:
                name += f": {lg_record[label]}"
        return name

    def parse_tag(self, tag):
        
        subtags = tag.split("-")
        lang = {}
        labels = ["language", "extlang", "script", "region", "variant", "variant"]
        while subtags and labels:
            subtag = subtags.pop(0)
            found = False
            while labels:
                label = labels.pop(0)
                subtag = self.casing[label](subtag)
                if self.format[label].fullmatch(subtag):
                    if subtag in self.db[label]:
                        found = True
                        valstr = self.val2str(self.db[label][subtag]["Description"])
                        if label == "variant" and label in lang:
                            lang[label] += ": " + valstr
                        else:
                            lang[label] = valstr
                        break
                    elif subtag in self.db["deprecated"][label]:
                        found = True
                        note = f"The {subtag!r} {label} code is deprecated"
                        if "Preferred-Value" in self.db["deprecated"][label][subtag]:
                            prefer = self.db["deprecated"][label][subtag][
                                "Preferred-Value"
                            ]
                            note += f"', prefer '{self.val2str(prefer)}'"
                        lang[label] = self.val2str(
                            self.db["deprecated"][label][subtag]["Description"]
                        )
                        warn(note)
                        break
            if not found:
                if subtag == "u" and subtags[0] == "sd":  
                    sd = subtags[1]
                    if sd in self.subdiv:
                        ext = self.subdiv[sd]
                    else:
                        ext = f"<Unknown subdivision: {ext}>"
                else:  
                    ext = f"{subtag}{''.join(['-'+ext for ext in subtags])}".lower()
                    if not self.format["singleton"].fullmatch(subtag):
                        ext = f"<Invalid extension: {ext}>"
                        warn(ext)
                lang["extension"] = ext
                subtags = []
        return lang

    def name(self, tag):
        
        for label in ["redundant", "grandfathered"]:
            val = None
            if tag in self.db[label]:
                val = f"{self.db[label][tag]['Description']}"
                note = f"The {tag!r} code is {label}"
            elif tag in self.db["deprecated"][label]:
                val = f"{self.db['deprecated'][label][tag]['Description']}"
                note = f"The {tag!r} code is {label} and deprecated"
                if "Preferred-Value" in self.db["deprecated"][label][tag]:
                    prefer = self.db["deprecated"][label][tag]["Preferred-Value"]
                    note += f", prefer {self.val2str(prefer)!r}"
            if val:
                warn(note)
                return val
        try:
            return self.lang2str(self.parse_tag(tag))
        except:
            warn(f"Tag {tag!r} was not recognized")
            return None