<jupyter_start><jupyter_text>Popular News Posts<jupyter_code>min_participants = 300
popular_news = news[news['participants']>=min_participants]
print "# of popular news posts:", len(popular_news)
stop_words = stopwords.words("english")
stop_words.extend(['coronavirus', 'covid', 'virus', 'news', 'new', 'outbreak', 'pandemic'])
# convert to lowercase
def text_lowercase(text): 
    return text.decode('ascii', 'ignore').lower() 

# remove numbers
def remove_numbers(text): 
    result = re.sub(r'\d+', '', text.encode('ascii', 'ignore')) 
    return result 

# remove whitespace from text 
def remove_whitespace(text): 
    return  " ".join(text.encode('ascii', 'ignore').split())

# remove stopwords 
def remove_stopwords(text): 
    word_tokens = word_tokenize(text.encode('ascii', 'ignore')) 
    filtered_text = [word for word in word_tokens if word not in stop_words] 
    return filtered_text 

# pre-process text
def pre_process(text):
    text_lower = text_lowercase(str(text).decode('ascii', 'ignore'))
    no_numbers = remove_numbers(text_lower)
    no_punctuation = no_numbers.translate(None, string.punctuation)
    no_whitespace = remove_whitespace(no_punctuation)
    no_stopwords = remove_stopwords(no_whitespace)
    lemmas = [lemmatizer.lemmatize(token) for token in no_stopwords]
    no_chars = [i for i in lemmas if len(i) > 1]
    return no_chars   

# get bi-grams
def bi_grams(tokens):
    return list(nltk.bigrams(tokens))
pool = multiprocessing.Pool(processes=8)
text = pool.map(pre_process, popular_news['text'].dropna())
words = [element for lis in text for element in lis]
print Counter(words).most_common(100) 
grams_collection = pool.map(bi_grams, text)
topics = []
for grams in tqdm_notebook(grams_collection):
    topics.extend(grams)
Counter(topics).most_common(100)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12800875273522977, 'fraction_numerical': 0.006017505470459519, 'mean_word_length': 5.579136690647482, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '452641', 'license': 'MIT', 'n_tokens_mistral': 651, 'n_tokens_neox': 603, 'n_words': 134, 'path': 'Notebooks/social_engagement_outliers.ipynb', 'repo_name': 'LINK-NU/JQD-DM-COVID-19', 'size': '1828'}