<jupyter_start><jupyter_text>Get google trending key words<jupyter_code># read more about pytrends API here: https://pypi.org/project/pytrends/
pytrend = TrendReq()
google_topics = (
    pytrend.
    trending_searches(pn='united_states').
    rename(columns={0:'topic'}).
    query('topic.str.split().str.len()<=3'). #only use trending topics at most 3 words
    topic.str.lower().
    reset_index(drop=True)
)
google_topics = set(google_topics.values)
google_topics<jupyter_output><empty_output><jupyter_text>get reddit post title tri-grams<jupyter_code>!ls ./data
path = Path('./data/worldnews_posts.csv')
reddit_posts = (
    pd.read_csv(path).
    iloc[1:,:].
    reset_index(drop=True) #the first title appears to be pinned post
)
reddit_posts
nlp=spacy.load('en_core_web_sm')
trigram_vectorizer=CountVectorizer(ngram_range=(1,3))
def text_processing(text: str, processor:spacy.lang.en.English) -> str:
    return ' '.join(i.lemma_.lower() for i in processor(text) 
                    if not (i.is_punct or i.is_stop or i.pos_ == 'VERB'))
def get_ngram(text:str,n_gram_min:int=1, n_gram_max:int=3) -> Set[str]:
    trigram_vectorizer = CountVectorizer(ngram_range=(n_gram_min,n_gram_max))
    trigram_vectorizer.fit_transform([text])
    return set(trigram_vectorizer.get_feature_names_out())
reddit_posts['ngram'] = (
     reddit_posts.title.
        apply(text_processing, args=(nlp,)).
        apply(get_ngram)
)<jupyter_output><empty_output><jupyter_text>Find what trending topic is mentioned in reddit post title<jupyter_code>reddit_posts.ngram[0]
topic_intersect = partial(set.intersection,google_topics)
counter = defaultdict(int)
for i in filter(lambda x: x, reddit_posts.ngram.apply(topic_intersect)):
    for topic in i:
        counter[topic]+=1
{topic:np.round(count/len(reddit_posts),3) for topic, count in counter.items()}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13247863247863248, 'fraction_numerical': 0.005876068376068376, 'mean_word_length': 5.349152542372881, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '317908', 'license': 'MIT', 'n_tokens_mistral': 686, 'n_tokens_neox': 682, 'n_words': 119, 'path': 'notebooks/Google_Trend_Demo - Luo 2-4-2022.ipynb', 'repo_name': 'reddit-conflicting-viewpoints/Reddit', 'size': '1872'}