<commit_before>import random
import re
import spacy
import string
import pronouncing
from interfaces.SentenceOperation import SentenceOperation
from tasks.TaskTypes import TaskType




def grapheme_phoneme(grapheme):
    '''
    converts each word to phonems. If there are multiple pronounciation of a
    word, only the first prononciation is taken. Stress information from each
    word is removed.
    '''
    phoenems = pronouncing.phones_for_word(grapheme)

    if len(phoenems)> 0:
        phoenem_without_stress = ''.join([x for x in phoenems[0] if x.isalpha()]).lower()
        transformed_word = phoenem_without_stress
    else:
        transformed_word = grapheme

    return transformed_word


def phoneme_substitution(text, spacy_pipeline, prob = 0.5, seed = 0, max_outputs = 1):
    random.seed(seed)

    '''
    returns transformed sentence with words replaced with their pronunciation
    based on probability.
    '''

    doc = spacy_pipeline(text)
    
    transformed_words = []
    for token in doc:
        word = token.text
        if word in string.punctuation:
            transformed_words[-1] += word
        else:
        
            if random.random() < prob:
                new_word = grapheme_phoneme(word)
                transformed_words.append(new_word)
            else:
                transformed_words.append(word)
    
    transformed_sentence = [' '.join(transformed_words)]

    return [transformed_sentence]



class PhonemeSubstitution(SentenceOperation):
    tasks = [
        TaskType.TEXT_CLASSIFICATION,
        TaskType.TEXT_TO_TEXT_GENERATION,
    ]
    languages = ["en"]

    def __init__(self, seed=0, prob=0.5, max_outputs=1):
        super().__init__(seed, max_outputs=max_outputs)
        self.spacy_pipeline = spacy.load("en_core_web_sm")
        self.prob = prob

    def generate(self, sentence: str):
        perturbed = phoneme_substitution(
            text=sentence,
            spacy_pipeline=self.spacy_pipeline,
            seed=self.seed,
            prob=self.prob,
            max_outputs=self.max_outputs,
        )
        return perturbed<commit_msg>updated spacy_pipeline<commit_after>import random
import re
import string

import pronouncing
import spacy

from initialize import spacy_nlp
from interfaces.SentenceOperation import SentenceOperation
from tasks.TaskTypes import TaskType


def grapheme_phoneme(grapheme):
    """
    converts each word to phonems. If there are multiple pronounciation of a
    word, only the first prononciation is taken. Stress information from each
    word is removed.
    """
    phoenems = pronouncing.phones_for_word(grapheme)

    if len(phoenems) > 0:
        phoenem_without_stress = "".join(
            [x for x in phoenems[0] if x.isalpha()]
        ).lower()
        transformed_word = phoenem_without_stress
    else:
        transformed_word = grapheme

    return transformed_word


def phoneme_substitution(
    text, spacy_pipeline, prob=0.5, seed=0, max_outputs=1
):
    random.seed(seed)

    """
    returns transformed sentence with words replaced with their pronunciation
    based on probability.
    """

    doc = spacy_pipeline(text)

    transformed_words = []
    for token in doc:
        word = token.text
        if word in string.punctuation:
            transformed_words[-1] += word
        else:

            if random.random() < prob:
                new_word = grapheme_phoneme(word)
                transformed_words.append(new_word)
            else:
                transformed_words.append(word)

    transformed_sentence = [" ".join(transformed_words)]

    return [transformed_sentence]


class PhonemeSubstitution(SentenceOperation):
    tasks = [
        TaskType.TEXT_CLASSIFICATION,
        TaskType.TEXT_TO_TEXT_GENERATION,
    ]
    languages = ["en"]

    def __init__(self, seed=0, prob=0.5, max_outputs=1):
        super().__init__(seed, max_outputs=max_outputs)
        self.spacy_pipeline = (
            spacy_nlp if spacy_nlp else spacy.load("en_core_web_sm")
        )
        self.prob = prob

    def generate(self, sentence: str):
        perturbed = phoneme_substitution(
            text=sentence,
            spacy_pipeline=self.spacy_pipeline,
            seed=self.seed,
            prob=self.prob,
            max_outputs=self.max_outputs,
        )
        return perturbed
{'fraction_non_alphanumeric': 0.07911683532658693, 'fraction_numerical': 0.005059797608095676, 'mean_word_length': 2.564754098360656, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2405308', 'n_tokens_mistral': 1328, 'n_tokens_neox': 1264, 'n_words': 302}