<jupyter_start><jupyter_text>Exploring natural language processing on Reddit commentsThe most important part of this notebook is the extraction of product names and types from the appropriate Reddit comment. I looked into various was of topic modeling, but because people often don't write out product names and sometimes the names are misspelled, I really had to hand-hold the process.<jupyter_code>import pandas as pd
import numpy as np
import re<jupyter_output><empty_output><jupyter_text>Drop duplicate rows and posts with no comments<jupyter_code>curly_df = pd.read_csv('matched_posts.csv')

# Drop duplicates and deleted comments
curly_df.drop_duplicates(subset='sub_id', keep = False, inplace = True) 
curly_df.dropna(subset=['comm_text'], inplace=True)
curly_df = curly_df[curly_df['comm_text'] != '[deleted]']
curly_df = curly_df[curly_df['comm_text'] != '[removed]]']
curly_df.index = range(len(curly_df))
text = [re.sub(r'http\S+', '', t) for t in curly_df['comm_text']] # remove links
text = [re.sub(r'([0-9]+?)', '', t).lower() for t in text] # remove all numbers and symbols
text = [re.sub(r'(!|#|\$|%|\(|\)|\*|\+|,|-|\.|/|:|;|<|=|>|\?|@|\[|\\|\]|\^|_|`|{|\||}|~)+', "", t)
        for t in text]
text = [re.sub(r"(')+", "", t) for t in text]
curly_df['comm_text'] = [re.sub(r'\s+\s', ' ', t).strip() for t in text] # replace double spaces with single spaces
products = pd.read_csv('./curly_products.csv', encoding='utf8')
products.rename(columns={"Unnamed: 0": "product", "Unnamed: 1": "type"}, inplace=True)
products.drop(columns=['Unnamed: 2','Unnamed: 3','Unnamed: 4'], inplace=True)

text = [re.sub(r'([0-1]+?)', '', t).lower() for t in products['product']] # remove all numbers and symbols
products['product'] = [re.sub(r'(\')+', '', t) for t in text]

products['brand']=[t[0] for t in products['product'].str.split()]
products['brand']=products['brand'].str.replace('_', ' ')

text = [re.sub(r'([0-1]+?)', ' ', t).lower() for t in products['brand']] # remove all numbers and symbols
products['brand'] = [re.sub(r"(')+", "", t) for t in text]
types_of_products = products['type'].dropna().unique()
types_of_products = np.append(types_of_products,['nopoo', 'no poo', 'no-poo', 'lopoo', 'lo poo', 'lo-poo','leavein', 
                          'leave in', 'cowash', 'co wash', 'co-wash','creme', 'lowpoo', 'low-poo', 'low poo', 
                            'condition','deep conditioner', 'styler', 'styling', 'milk', 'mask'])
print(types_of_products)
curly_df['products_used'] = ''

for i,text in enumerate(curly_df['comm_text']):
    prod = []

    for j,brand in enumerate(products['brand']):
        if re.search(r'\b'+str(brand)+r'\b', str(text)):
            prod.append((str(products['product'].iloc[j]), products['type'].iloc[j]))
            
    prod = list(dict.fromkeys(prod))
    curly_df['products_used'].iloc[i] = prod
    
curly_df['products_used'].iloc[-2]
curly_df.head(10)

# Save the products extracted from each comment describing the poster's routine to a file
curly_df.to_csv('dataframe_with_products.csv')<jupyter_output><empty_output><jupyter_text>Topic modeling via LDA using spaCy and gensim<jupyter_code># Removing stop words not necessary for my purposes

import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')
stop = stopwords.words('english')

curly_df['comm_text'] = curly_df['comm_text'].apply(lambda x: [item for item in x if item not in stop])
#! python -m spacy download en_core_web_sm
nlp = spacy.load('en_core_web_sm')
texts = curly_df['comm_text'].tolist()
%time spacy_docs = list(nlp.pipe(texts))
docs = [[t.lemma_.lower() for t in doc if len(t.orth_) > 2 and not t.is_stop] for doc in nlp]
from gensim.models import Phrases

bigram = Phrases(docs, min_count=10)
tokens = []

for idx in range(len(docs)):
    for token in bigram[docs[idx]]:
        if '_' in token:  # bigrams can be recognized by the "_" that joins the invidual words
            docs[idx].append(token)
            tokens.append(token)
            
print(list(set(tokens)))
from gensim.corpora import Dictionary

dictionary = Dictionary(docs)
print('Number of unique words in original documents:', len(dictionary))

dictionary.filter_extremes(no_below=3, no_above=0.25)
print('Number of unique words after removing rare and common words:', len(dictionary))

print("Example representation of document 3:", dictionary.doc2bow(docs[2]))
corpus = [dictionary.doc2bow(doc) for doc in docs]
from gensim.models import LdaModel

%time model = LdaModel(corpus=corpus, id2word=dictionary, num_topics=10, chunksize=500, passes=3, random_state=1)
for (topic, words) in model.print_topics():
    print(topic+1, ":", words, '\n')
import pyLDAvis.gensim
import warnings

pyLDAvis.enable_notebook()
warnings.filterwarnings("ignore", category=DeprecationWarning) 

pyLDAvis.gensim.prepare(model, corpus, dictionary, sort_topics=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16905151267375307, 'fraction_numerical': 0.007358953393295176, 'mean_word_length': 5.201520912547529, 'pattern_counts': {'":': 3, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '293307', 'license': 'MIT', 'n_tokens_mistral': 1724, 'n_tokens_neox': 1668, 'n_words': 432, 'path': 'notebooks/exploratory_comment_NLP.ipynb', 'repo_name': 'jsbridge/makemecurly', 'size': '4892'}