<jupyter_start><jupyter_text>Article spinner<jupyter_code>from bs4 import BeautifulSoup
import numpy as np<jupyter_output><empty_output><jupyter_text>We'll use positive electonics reviews as our dataset. Reviews will be parsed into trigrams, computing the probability of the middle word given the previous and the next word in the review.<jupyter_code>reviews = BeautifulSoup(open('../data/sorted_data_acl/electronics/positive.review').read())
reviews = [tag.text for tag in reviews.findAll('review_text')]
len(reviews)<jupyter_output><empty_output><jupyter_text>Very lightweight sentence cleanup that will remove unnecessary characters that are not helpful for our model.<jupyter_code>import re

def clean_text(t):
    words = re.split('; |, |\*|\n|/|_', t)
    words = [word.strip(',.:-_').lower() for word in words]
    words = [word for word in words if len(word) > 0]
    return ' '.join(words)

reviews = [clean_text(review) for review in reviews]
reviews = [review.split(' ') for review in reviews]<jupyter_output><empty_output><jupyter_text>At first, we count occurences of words given the previous and next word in the sentence. Then, we re-compute those counts into the conditional probabilities.<jupyter_code>def get_count_trigrams(reviews):
    trigrams = {}
    for review in reviews:
        for idx in range(1, len(review) - 1):
            key = (review[idx-1], review[idx+1])
            if key not in trigrams:
                trigrams[key] = {}

            word = review[idx]
            if review[idx] not in trigrams[key]:
                trigrams[key][word] = 1
            else:
                trigrams[key][word] += 1
    return trigrams

def get_probability_trigrams(reviews):
    trigrams = get_count_trigrams(reviews)
    
    for key in trigrams:
        s = sum(trigrams[key].values())
        for words in trigrams[key]:
            trigrams[key][words] /= s
    return trigrams<jupyter_output><empty_output><jupyter_text>Pick word method will return a word with a respect to the given probabilities.<jupyter_code>trigrams = get_probability_trigrams(reviews)
trigrams[('i', 'this')].values()
def pick_word(word_probs):
    return np.random.choice(list(word_probs.keys()), 1, p=list(word_probs.values()))[0]
sum(trigrams[('i', 'this')].values()), pick_word(trigrams[('i', 'this')])<jupyter_output><empty_output><jupyter_text>We can now 'spin' an article. Words will be replaced randomly with recpect to the probability of their occurence given the previous and the next word.<jupyter_code>def spin(article):
    article = article.lower()
    words = article.split(' ')
    for i in range(1, len(words) - 2):
        if np.random.random() < 0.5:
            p = words[i-1]
            n = words[i+1]
            
            if (p, n) in trigrams:
                words[i] = pick_word(trigrams[(p, n)])
    
    return ' '.join(words)
article = ' '.join(np.random.choice(reviews))
new_review = spin(article)

article, new_review<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11981258366800536, 'fraction_numerical': 0.00535475234270415, 'mean_word_length': 3.6849529780564265, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '36975', 'license': 'MIT', 'n_tokens_mistral': 1011, 'n_tokens_neox': 937, 'n_words': 260, 'path': 'nlp/Article Spinner.ipynb', 'repo_name': 'ta-ja/Machine-Learning-Journey-Notebooks', 'size': '2988'}