<commit_before>import spacy

from initialize import spacy_nlp
from interfaces.SentenceOperation import SentenceOperation
from interfaces.SentencePairOperation import SentencePairOperation
from tasks.TaskTypes import TaskType

"""
Auxiliary Negation Removal.
    Remove auxiliary negations generating a sentence with oposite meaning.
"""


def auxiliary_negation_removal(sentence, nlp):

    # Tokenize Sentence
    doc = nlp(sentence)

    # Initialize Variables
    changed = False
    new_sentence = []
    supported_auxiliaries = [
        "am",
        "are",
        "can",
        "could",
        "had",
        "has",
        "have",
        "is",
        "may",
        "might",
        "must",
        "shall",
        "should",
        "was",
        "were",
        "will",
        "would",
    ]

    # Evaluate Tokens
    for token in doc:
        # Add Token to Output Sentence
        new_sentence.append(token)

        # Initialize Variables
        token_lowercased_lemma = token.lemma_.lower()

        # Process Negations
        if token_lowercased_lemma == "not" or token_lowercased_lemma == "n't":
            # Get not position
            not_index = token.i

            # Process Auxiliaries
            if not_index > 0:

                # Get Previous Token
                previous_index = not_index - 1
                previous_token = doc[previous_index]
                previous_surface = previous_token.text
                previous_lowercase_surface = previous_surface.lower()

                # Remove Negation
                if previous_lowercase_surface in supported_auxiliaries:
                    new_sentence = new_sentence[:-1]
                    changed = True

                elif previous_lowercase_surface in ["do"]:
                    new_sentence = new_sentence[:-2]
                    changed = True

                # Handle Spacing
                if (
                    token_lowercased_lemma == "n't"
                    and changed
                    and new_sentence
                ):
                    new_sentence[-1] = nlp(
                        new_sentence[-1].text + token.whitespace_
                    )[0]

    # Rebuild Sentence
    new_sentence = [t.text + t.whitespace_ for t in new_sentence]
    new_sentence = "".join(new_sentence)

    return (new_sentence, changed)


class SentenceAuxiliaryNegationRemoval(SentenceOperation):
    tasks = [
        TaskType.TEXT_CLASSIFICATION,
        TaskType.TEXT_TO_TEXT_GENERATION,
        TaskType.QUALITY_ESTIMATION,
    ]
    languages = ["en"]
    keywords = [
        "morphological",
        "lexical",
        "rule-based",
        "tokenizer-required",
        "meaning-alteration",
        "high-precision",
        "low-coverage",
        "low-generations",
        "causal-reasoning",
    ]

    def __init__(self, seed=0, max_outputs=1):
        super().__init__(seed, max_outputs=max_outputs)
        self.nlp = spacy_nlp if spacy_nlp else spacy.load("en_core_web_sm")

    def generate(self, sentence: str):

        # Initialize Variables
        output_sentence = sentence

        # Process sentence
        new_sentence, changed = auxiliary_negation_removal(sentence, self.nlp)

        if changed:
            output_sentence = new_sentence

        return [output_sentence]


class PairAuxiliaryNegationRemoval(SentencePairOperation):
    tasks = [TaskType.PARAPHRASE_DETECTION]
    languages = ["en"]
    keywords = [
        "morphological",<commit_msg>Apply spacy tokenizer reinitialization<commit_after>import spacy

from initialize import spacy_nlp

from spacy.util import compile_prefix_regex, compile_suffix_regex, compile_infix_regex
from spacy.tokenizer import Tokenizer

from interfaces.SentenceOperation import SentenceOperation
from interfaces.SentencePairOperation import SentencePairOperation
from tasks.TaskTypes import TaskType

"""
Auxiliary Negation Removal.
    Remove auxiliary negations generating a sentence with oposite meaning.
"""


def auxiliary_negation_removal(sentence, nlp):

    # Tokenize Sentence
    doc = nlp(sentence)

    # Initialize Variables
    changed = False
    new_sentence = []
    supported_auxiliaries = [
        "am",
        "are",
        "can",
        "could",
        "had",
        "has",
        "have",
        "is",
        "may",
        "might",
        "must",
        "shall",
        "should",
        "was",
        "were",
        "will",
        "would",
    ]

    # Evaluate Tokens
    for token in doc:
        # Add Token to Output Sentence
        new_sentence.append(token)

        # Initialize Variables
        token_lowercased_lemma = token.lemma_.lower()

        # Process Negations
        if token_lowercased_lemma == "not" or token_lowercased_lemma == "n't":
            # Get not position
            not_index = token.i

            # Process Auxiliaries
            if not_index > 0:

                # Get Previous Token
                previous_index = not_index - 1
                previous_token = doc[previous_index]
                previous_surface = previous_token.text
                previous_lowercase_surface = previous_surface.lower()

                # Remove Negation
                if previous_lowercase_surface in supported_auxiliaries:
                    new_sentence = new_sentence[:-1]
                    changed = True

                elif previous_lowercase_surface in ["do"]:
                    new_sentence = new_sentence[:-2]
                    changed = True

                # Handle Spacing
                if (
                    token_lowercased_lemma == "n't"
                    and changed
                    and new_sentence
                ):
                    new_sentence[-1] = nlp(
                        new_sentence[-1].text + token.whitespace_
                    )[0]

    # Rebuild Sentence
    new_sentence = [t.text + t.whitespace_ for t in new_sentence]
    new_sentence = "".join(new_sentence)

    return (new_sentence, changed)


class SentenceAuxiliaryNegationRemoval(SentenceOperation):
    tasks = [
        TaskType.TEXT_CLASSIFICATION,
        TaskType.TEXT_TO_TEXT_GENERATION,
        TaskType.QUALITY_ESTIMATION,
    ]
    languages = ["en"]
    keywords = [
        "morphological",
        "lexical",
        "rule-based",
        "tokenizer-required",
        "meaning-alteration",
        "high-precision",
        "low-coverage",
        "low-generations",
        "causal-reasoning",
    ]

    def __init__(self, seed=0, max_outputs=1):
        super().__init__(seed, max_outputs=max_outputs)
        
        nlp = spacy_nlp if spacy_nlp else spacy.load('en_core_web_sm')
        rules = nlp.Defaults.tokenizer_exceptions
        infix_re = compile_infix_regex(nlp.Defaults.infixes)
        prefix_re = compile_prefix_regex(nlp.Defaults.prefixes)
        suffix_re = compile_suffix_regex(nlp.Defaults.suffixes)

        nlp.tokenizer = Tokenizer(
            nlp.vocab,
            rules = rules,
            prefix_search=prefix_re.search,
            suffix_search=suffix_re.search,
            infix_finditer=infix_re.finditer,
        )
        
        self.nlp = nlp

    def generate(self, sentence: str):

        # Initialize Variables
        output_sentence = sentence

        # Process sentence
        new_sentence, changed = auxiliary_negation_removal(sentence, self.nlp)

        if changed:
            output_sentence = new_sentence

        return [output_sentence]


class PairAuxiliaryNegationRemoval(SentencePairOperation):
    tasks = [TaskType.PARAPHRASE_DETECTION]
    languages = ["en"]
    keywords = [
        "morphological",
{'fraction_non_alphanumeric': 0.08095051573312442, 'fraction_numerical': 0.0023501762632197414, 'mean_word_length': 1.9125475285171103, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5136742', 'n_tokens_mistral': 2271, 'n_tokens_neox': 2079, 'n_words': 450}