<commit_before>import six
from enum import Enum
from yaml import dump

"""
This file contains all models.
"""


class Corpus:
    """
    The class representing a corpus files.

    This class represents the 1-1 mapping to and from the tc-xml files.
    """
    def __init__(self):
        self.texts = []

    def __iter__(self):
        return self.texts.__iter__()

    def to_dict(self):
        return {
            'texts': self.texts
        }


class Text:
    """
    The class representing a text-object.

    A text is formally a collection of sentences with some extra metadata.
    """

    def __init__(
        self,
        title="",
        title_translation="",
        language="und",
        plain_text="",
        rich_text="",
        metadata=None,
        phrases=None
    ):
        """
        Default constructor
        """
        self.title = title
        self.title_translation = title_translation
        self.language = language  # Und is the default for the undefined language
        self.plain_text = plain_text
        self.rich_text = rich_text
        self.metadata = {}
        self.phrases = []

        if phrases and hasattr(phrases, '__iter__'):
            self.add_phrases(phrases)

        if metadata and isinstance(metadata, dict):
            self.metadata.update(metadata)

    def add_phrase(self, phrase):
        """
        Adds a phrase to the text-object

        :param phrase:
        :return:
        """
        if not (isinstance(phrase, Phrase)):
            raise Exception("Wrong argument to add_phrase. Expected Phrase instance")

        self.phrases.append(phrase)

    def add_phrases(self, phrases):
        """
        Adds an iterable of phrases to this text.

        :param phrases:
        :return:
        """
        for phrase in phrases:
            self.add_phrase(phrase)

    def add_metadata(self, key, value):
        """
        Adds a metadata key-value pair.

        :param key:
        :param value:
        :return:
        """

        if key is not None and value is not None:
            self.metadata[key] = value
        else:
            raise Exception("Wrong argument to add_metadata. Expected a key-value pair as argument one and two")

    def remove_phrase(self, phrase):
        """
        Removes a phrase from the text if it exists.

        :param phrase:
        :return:
        """
        self.phrases.remove(phrase)

    def remove_metadata(self, key):
        """
        Deletes a metadata key from the texts metadata dict if it exists.
        :param key:
        :return:
        """
        if key in self.metadata:
            del self.metadata[key]

    def clear_phrases(self):
        """
        Clears all phrases of the text.

        :return:
        """
        self.phrases = []

    def clear_metadata(self):
        """
        Clears all metadata of the text.

        :return:
        """
        self.metadata = {}

    def clear_tags(self):
        """
        Clears all tags in the text. Specifically, call clear_tags on all the phrases of the text
        :return:
        """
        for phrase in self.phrases:
            phrase.clear_tags()

    def attributes(self):
        """
        Return all non-children attributes of the text.

        :return:
        """
        return {
            'title': self.title,
            'title_translation': self.title_translation,
            'language': self.language,
            'plain_text': self.plain_text,
            'rich_text': self.rich_text,
            'metadata': self.metadata,
        }

    def to_dict(self):
        return {
            'title': self.title,
            'title_translation': self.title_translation,
            'language': self.language,
            'plain_text': self.plain_text,
            'rich_text': self.rich_text,
            'metadata': self.metadata,
            'phrases': list(map(lambda phr: phr.to_dict(), self.phrases))
        }

    def __str__(self):
        return dump(self.to_dict())

    def __iter__(self):
        return self.phrases.__iter__()


class PhraseValidity(Enum):
    UNKNOWN = 1
    VALID = 2
    INVALID = 3
    SPECIAL = 4
    EMPTY = 5


class Phrase:
    """<commit_msg>Change PhraseValidity enum values to be strings<commit_after>import six
from enum import Enum
from yaml import dump

"""
This file contains all models.
"""


class Corpus:
    """
    The class representing a corpus files.

    This class represents the 1-1 mapping to and from the tc-xml files.
    """
    def __init__(self):
        self.texts = []

    def __iter__(self):
        return self.texts.__iter__()

    def to_dict(self):
        return {
            'texts': self.texts
        }


class Text:
    """
    The class representing a text-object.

    A text is formally a collection of sentences with some extra metadata.
    """

    def __init__(
        self,
        title="",
        title_translation="",
        language="und",
        plain_text="",
        rich_text="",
        metadata=None,
        phrases=None
    ):
        """
        Default constructor
        """
        self.title = title
        self.title_translation = title_translation
        self.language = language  # Und is the default for the undefined language
        self.plain_text = plain_text
        self.rich_text = rich_text
        self.metadata = {}
        self.phrases = []

        if phrases and hasattr(phrases, '__iter__'):
            self.add_phrases(phrases)

        if metadata and isinstance(metadata, dict):
            self.metadata.update(metadata)

    def add_phrase(self, phrase):
        """
        Adds a phrase to the text-object

        :param phrase:
        :return:
        """
        if not (isinstance(phrase, Phrase)):
            raise Exception("Wrong argument to add_phrase. Expected Phrase instance")

        self.phrases.append(phrase)

    def add_phrases(self, phrases):
        """
        Adds an iterable of phrases to this text.

        :param phrases:
        :return:
        """
        for phrase in phrases:
            self.add_phrase(phrase)

    def add_metadata(self, key, value):
        """
        Adds a metadata key-value pair.

        :param key:
        :param value:
        :return:
        """

        if key is not None and value is not None:
            self.metadata[key] = value
        else:
            raise Exception("Wrong argument to add_metadata. Expected a key-value pair as argument one and two")

    def remove_phrase(self, phrase):
        """
        Removes a phrase from the text if it exists.

        :param phrase:
        :return:
        """
        self.phrases.remove(phrase)

    def remove_metadata(self, key):
        """
        Deletes a metadata key from the texts metadata dict if it exists.
        :param key:
        :return:
        """
        if key in self.metadata:
            del self.metadata[key]

    def clear_phrases(self):
        """
        Clears all phrases of the text.

        :return:
        """
        self.phrases = []

    def clear_metadata(self):
        """
        Clears all metadata of the text.

        :return:
        """
        self.metadata = {}

    def clear_tags(self):
        """
        Clears all tags in the text. Specifically, call clear_tags on all the phrases of the text
        :return:
        """
        for phrase in self.phrases:
            phrase.clear_tags()

    def attributes(self):
        """
        Return all non-children attributes of the text.

        :return:
        """
        return {
            'title': self.title,
            'title_translation': self.title_translation,
            'language': self.language,
            'plain_text': self.plain_text,
            'rich_text': self.rich_text,
            'metadata': self.metadata,
        }

    def to_dict(self):
        return {
            'title': self.title,
            'title_translation': self.title_translation,
            'language': self.language,
            'plain_text': self.plain_text,
            'rich_text': self.rich_text,
            'metadata': self.metadata,
            'phrases': list(map(lambda phr: phr.to_dict(), self.phrases))
        }

    def __str__(self):
        return dump(self.to_dict())

    def __iter__(self):
        return self.phrases.__iter__()


class PhraseValidity(Enum):
    UNKNOWN = 'UNKNOWN'
    VALID = 'VALID'
    INVALID = 'INVALID'
    SPECIAL = 'SPECIAL'
    EMPTY = 'EMPTY'


class Phrase:
    """
{'fraction_non_alphanumeric': 0.09784138901923979, 'fraction_numerical': 0.001055842327545753, 'mean_word_length': 1.7508873830267828, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2330957', 'n_tokens_mistral': 2418, 'n_tokens_neox': 2342, 'n_words': 694}