<commit_before>"""Custom data types for the CLTK. These types form the building blocks
of the NLP pipeline.


>>> from cltkv1.core.data_types import Language
>>> from cltkv1.core.data_types import Word
>>> from cltkv1.core.data_types import Process
>>> from cltkv1.core.data_types import Doc
>>> from cltkv1.core.data_types import Pipeline
"""

from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Type, Union


@dataclass
class Language:
    """For holding information about any given language. Used to
    encode data from ISO 639-3 and Glottolog at
    ``cltkv1.lagnuages.glottolog.LANGUAGES`` May be extended by
    user for dialects or languages not documented by ISO 639-3.

    >>> from cltkv1.core.data_types import Language
    >>> from cltkv1.languages.utils import get_lang
    >>> latin = get_lang("lat")
    >>> isinstance(latin, Language)
    True
    >>> latin
    Language(name='Latin', glottolog_id='lati1261', latitude=41.9026, longitude=12.4502, dates=[], family_id='indo1319', parent_id='impe1234', level='language', iso_639_3_code='lat', type='a')
    """

    name: str  # Glottolog description
    glottolog_id: str
    latitude: float
    longitude: float
    dates: List[int]  # add later; not available from Glottolog or ISO list
    family_id: str  # from Glottolog
    parent_id: str  # from Glottolog
    level: str  # a language or a dialect
    iso_639_3_code: str
    type: str  # "a" for ancient and "h" for historical; this from Glottolog


@dataclass
class Word:
    """Contains attributes of each processed word in a list of
    words. Designed to be used in the ``Doc.words`` dataclass.

    >>> from cltkv1.core.data_types import Word
    >>> from cltkv1.utils.example_texts import get_example_text
    >>> get_example_text("lat")[:25]
    'Gallia est omnis divisa i'
    >>> from cltkv1.languages.utils import get_lang
    >>> latin = get_lang("lat")
    >>> Word(index_char_start=0, index_char_stop=6, index_token=0, string=get_example_text("lat")[0:6], pos="nom")
    Word(index_char_start=0, index_char_stop=6, index_token=0, index_sentence=None, string='Gallia', pos='nom', lemma=None, scansion=None, xpos=None, upos=None, dependency_relation=None, governor=None, parent=None, features=None)
    """

    index_char_start: int = None
    index_char_stop: int = None
    index_token: int = None
    index_sentence: int = None
    string: str = None
    pos: str = None
    lemma: str = None
    scansion: str = None
    xpos: str = None  # treebank-specific POS tag (from stanfordnlp)
    upos: str = None  # universal POS tag (from stanfordnlp)
    dependency_relation: str = None  # (from stanfordnlp)
    governor: "Word" = None
    parent: "Word" = None
    features: Dict[str, str] = None  # morphological features (from stanfordnlp)


@dataclass
class Doc:
    """The object returned to the user from the ``NLP()`` class.
    Contains overall attributes of submitted texts, plus most
    importantly the processed tokenized text ``words``,
    being a list of ``Word`` types.

    >>> from cltkv1 import NLP
    >>> from cltkv1.utils.example_texts import get_example_text
    >>> cltk_nlp = NLP(language="lat")
    >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
    >>> cltk_doc.raw[:38]
    'Gallia est omnis divisa in partes tres'
    >>> isinstance(cltk_doc.raw, str)
    True
    """

    language: str = None
    words: List[Word] = None
    pipeline: "Pipeline" = None
    raw: str = None

    @property
    def sentences(self) -> List[List[Word]]:
        sentences = {}
        for word in self.words:
            sentence = sentences.get(word.index_sentence, {})
            sentence[word.index_token] = word
            sentences[word.index_sentence] = sentence

        sorted_values = lambda dict: [x[1] for x in sorted(dict.items())]

        return [sorted_values(sentence) for sentence in sorted_values(sentences)]

    def _get_words_attribute(self, attribute):
        return [getattr(word, attribute) for word in self.words]

    @property
    def tokens(self) -> List[str]:
        """Returns a list of string word tokens of all words in the doc.

        >>> from cltkv1 import NLP
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> cltk_nlp = NLP(language="lat")
        >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
        >>> cltk_doc.tokens[:10]
        ['Gallia', 'est', 'omnis', 'divisa', 'in', 'partes', 'tres', ',', 'quarum', 'unam']
        """
        return self._get_words_attribute("string")

    @property
    def pos(self) -> List[str]:
        """Returns a list of the POS tags of all words in the doc.

        >>> from cltkv1 import NLP
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> cltk_nlp = NLP(language="lat")
        >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
        >>> cltk_doc.pos[:3]
        ['NOUN', 'AUX', 'DET']
        """
        return self._get_words_attribute("upos")

    @property
    def morphosyntactic_features(self) -> Dict[str, str]:
        """Returns a list of dictionaries containing the morphosyntactic features
        of each word (when available).
        Each dictionary specifies feature names as keys and feature values as values.

        >>> from cltkv1 import NLP
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> cltk_nlp = NLP(language="lat")
        >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
        >>> cltk_doc.morphosyntactic_features[:3]
        [{'Case': 'Nom', 'Degree': 'Pos', 'Gender': 'Fem', 'Number': 'Sing'}, {'Mood': 'Ind', 'Number': 'Sing', 'Person': '3', 'Tense': 'Pres', 'VerbForm': 'Fin', 'Voice': 'Act'}, {'Case': 'Nom', 'Degree': 'Pos', 'Gender': 'Fem', 'Number': 'Sing', 'PronType': 'Ind'}]
        """
        return self._get_words_attribute("features")

    @property
    def lemmata(self) -> List[str]:
        """Returns a list of lemmata, indexed to the word tokens
        provided by `Doc.tokens`.

        >>> from cltkv1 import NLP
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> cltk_nlp = NLP(language="lat")
        >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
        >>> cltk_doc.lemmata[:5]
        ['aallius', 'sum', 'omnis', 'divido', 'in']
        """
        return self._get_words_attribute("lemma")


@dataclass
class Process:
    """For each type of NLP process there needs to be a definition.
    It includes the type of data it expects (``str``, ``List[str]``,
    ``Word``, etc.) and what field within ``Word`` it will populate.
    This base class is intended to be inherited by NLP process
    types (e.g., ``TokenizationProcess`` or ``DependencyProcess``).

    >>> a_process = Process(input_doc=Doc(raw="input words here"))
    """

    input_doc: Doc
    output_doc: Doc = None
    algorithm = None
    language: str = None

    def run(self) -> None:<commit_msg>add lame embeddings method to Doc.<commit_after>"""Custom data types for the CLTK. These types form the building blocks
of the NLP pipeline.


>>> from cltkv1.core.data_types import Language
>>> from cltkv1.core.data_types import Word
>>> from cltkv1.core.data_types import Process
>>> from cltkv1.core.data_types import Doc
>>> from cltkv1.core.data_types import Pipeline
"""

from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Type, Union


@dataclass
class Language:
    """For holding information about any given language. Used to
    encode data from ISO 639-3 and Glottolog at
    ``cltkv1.lagnuages.glottolog.LANGUAGES`` May be extended by
    user for dialects or languages not documented by ISO 639-3.

    >>> from cltkv1.core.data_types import Language
    >>> from cltkv1.languages.utils import get_lang
    >>> latin = get_lang("lat")
    >>> isinstance(latin, Language)
    True
    >>> latin
    Language(name='Latin', glottolog_id='lati1261', latitude=41.9026, longitude=12.4502, dates=[], family_id='indo1319', parent_id='impe1234', level='language', iso_639_3_code='lat', type='a')
    """

    name: str  # Glottolog description
    glottolog_id: str
    latitude: float
    longitude: float
    dates: List[int]  # add later; not available from Glottolog or ISO list
    family_id: str  # from Glottolog
    parent_id: str  # from Glottolog
    level: str  # a language or a dialect
    iso_639_3_code: str
    type: str  # "a" for ancient and "h" for historical; this from Glottolog


@dataclass
class Word:
    """Contains attributes of each processed word in a list of
    words. Designed to be used in the ``Doc.words`` dataclass.

    >>> from cltkv1.core.data_types import Word
    >>> from cltkv1.utils.example_texts import get_example_text
    >>> get_example_text("lat")[:25]
    'Gallia est omnis divisa i'
    >>> from cltkv1.languages.utils import get_lang
    >>> latin = get_lang("lat")
    >>> Word(index_char_start=0, index_char_stop=6, index_token=0, string=get_example_text("lat")[0:6], pos="nom")
    Word(index_char_start=0, index_char_stop=6, index_token=0, index_sentence=None, string='Gallia', pos='nom', lemma=None, scansion=None, xpos=None, upos=None, dependency_relation=None, governor=None, parent=None, features=None)
    """

    index_char_start: int = None
    index_char_stop: int = None
    index_token: int = None
    index_sentence: int = None
    string: str = None
    pos: str = None
    lemma: str = None
    scansion: str = None
    xpos: str = None  # treebank-specific POS tag (from stanfordnlp)
    upos: str = None  # universal POS tag (from stanfordnlp)
    dependency_relation: str = None  # (from stanfordnlp)
    governor: "Word" = None
    parent: "Word" = None
    features: Dict[str, str] = None  # morphological features (from stanfordnlp)


@dataclass
class Doc:
    """The object returned to the user from the ``NLP()`` class.
    Contains overall attributes of submitted texts, plus most
    importantly the processed tokenized text ``words``,
    being a list of ``Word`` types.

    >>> from cltkv1 import NLP
    >>> from cltkv1.utils.example_texts import get_example_text
    >>> cltk_nlp = NLP(language="lat")
    >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
    >>> cltk_doc.raw[:38]
    'Gallia est omnis divisa in partes tres'
    >>> isinstance(cltk_doc.raw, str)
    True
    """

    language: str = None
    words: List[Word] = None
    pipeline: "Pipeline" = None
    raw: str = None

    @property
    def sentences(self) -> List[List[Word]]:
        sentences = {}
        for word in self.words:
            sentence = sentences.get(word.index_sentence, {})
            sentence[word.index_token] = word
            sentences[word.index_sentence] = sentence

        sorted_values = lambda dict: [x[1] for x in sorted(dict.items())]

        return [sorted_values(sentence) for sentence in sorted_values(sentences)]

    def _get_words_attribute(self, attribute):
        return [getattr(word, attribute) for word in self.words]

    @property
    def tokens(self) -> List[str]:
        """Returns a list of string word tokens of all words in the doc.

        >>> from cltkv1 import NLP
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> cltk_nlp = NLP(language="lat")
        >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
        >>> cltk_doc.tokens[:10]
        ['Gallia', 'est', 'omnis', 'divisa', 'in', 'partes', 'tres', ',', 'quarum', 'unam']
        """
        return self._get_words_attribute("string")

    @property
    def pos(self) -> List[str]:
        """Returns a list of the POS tags of all words in the doc.

        >>> from cltkv1 import NLP
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> cltk_nlp = NLP(language="lat")
        >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
        >>> cltk_doc.pos[:3]
        ['NOUN', 'AUX', 'DET']
        """
        return self._get_words_attribute("upos")

    @property
    def morphosyntactic_features(self) -> Dict[str, str]:
        """Returns a list of dictionaries containing the morphosyntactic features
        of each word (when available).
        Each dictionary specifies feature names as keys and feature values as values.

        >>> from cltkv1 import NLP
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> cltk_nlp = NLP(language="lat")
        >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
        >>> cltk_doc.morphosyntactic_features[:3]
        [{'Case': 'Nom', 'Degree': 'Pos', 'Gender': 'Fem', 'Number': 'Sing'}, {'Mood': 'Ind', 'Number': 'Sing', 'Person': '3', 'Tense': 'Pres', 'VerbForm': 'Fin', 'Voice': 'Act'}, {'Case': 'Nom', 'Degree': 'Pos', 'Gender': 'Fem', 'Number': 'Sing', 'PronType': 'Ind'}]
        """
        return self._get_words_attribute("features")

    @property
    def lemmata(self) -> List[str]:
        """Returns a list of lemmata, indexed to the word tokens
        provided by `Doc.tokens`.

        >>> from cltkv1 import NLP
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> cltk_nlp = NLP(language="lat")
        >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
        >>> cltk_doc.lemmata[:5]
        ['aallius', 'sum', 'omnis', 'divido', 'in']
        """
        return self._get_words_attribute("lemma")

    @property
    def embeddings(self):
        """Returns an embedding for each word.

        TODO: Consider option to use lemma

        >>> from cltkv1 import NLP
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> cltk_nlp = NLP(language="lat")
        >>> cltk_doc = cltk_nlp.analyze(text=get_example_text("lat"))
        >>> type(cltk_doc.embeddings[0])
        <class 'numpy.ndarray'>
        """
        from cltkv1.embeddings.fasttext_module import load_fasttext_model
        from cltkv1.embeddings.fasttext_module import get_fasttext_embedding

        ft_model = load_fasttext_model(
            iso_code="lat", training_set="wiki", model_type="bin"
        )
        return [
            get_fasttext_embedding(word=token, model=ft_model) for token in self.tokens
        ]


@dataclass
class Process:
    """For each type of NLP process there needs to be a definition.
    It includes the type of data it expects (``str``, ``List[str]``,
    ``Word``, etc.) and what field within ``Word`` it will populate.
    This base class is intended to be inherited by NLP process
    types (e.g., ``TokenizationProcess`` or ``DependencyProcess``).

    >>> a_process = Process(input_doc=Doc(raw="input words here"))
    """

    input_doc: Doc
    output_doc: Doc = None
    algorithm = None
    language: str = None

    def run(self) -> None:
{'fraction_non_alphanumeric': 0.12751723205838628, 'fraction_numerical': 0.011150155426408974, 'mean_word_length': 3.267301038062284, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 289, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4414407', 'n_tokens_mistral': 4865, 'n_tokens_neox': 4706, 'n_words': 1338}