<jupyter_start><jupyter_text>Reddit Cleaning 2  Import dependencies<jupyter_code>import pandas as pd
import re
import string

import pickle # just in case<jupyter_output><empty_output><jupyter_text>Set file locations<jupyter_code># File for cleaned comment data (input file)
cleaned_reddit_comments = '../data/reddit/reddit_comments_cleaned.csv'

# File for preprocessed comment text (output file)
cleaned_reddit_comment_text = '../data/reddit/reddit_comment_text_cleaned.csv'<jupyter_output><empty_output><jupyter_text>Read in the data<jupyter_code>df = pd.read_csv(cleaned_reddit_comments)
df.head()
df.info(verbose=True)<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 1911 entries, 0 to 1910
Data columns (total 10 columns):
 #   Column              Non-Null Count  Dtype  
---  ------              --------------  -----  
 0   post_id             1911 non-null   object 
 1   comment_id          1911 non-null   object 
 2   author              1911 non-null   object 
 3   comment             1911 non-null   object 
 4   created_utc         1911 non-null   float64
 5   downs               1911 non-null   float64
 6   ups                 1911 non-null   float64
 7   reply               1911 non-null   object 
 8   comment_replied_id  214 non-null    object 
 9   comment_date        1911 non-null   object 
dtypes: float64(3), object(7)
memory usage: 149.4+ KB<jupyter_text>Preprocess comment text for analysis<jupyter_code># Create a new dataframe. Use 'comment_id' for any future merges.
text_df = df[['comment_id', 'comment']].copy()
text_df.set_index('comment_id')
# Define a little cleaner function
# I would really like to get some review on the regex here.

def clean_text_round1(text):
    '''Make text lowercase, remove punctuation, excess whitespace (in that order).'''
    # make text lowercase
    text = text.lower()
    # remove punctuation
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    # remove multiple whitespace, and convert all whitespace to space (' ').
    text = " ".join(text.split())
    
    return text
text_df['clean_text'] = text_df['comment'].apply(lambda x: clean_text_round1(x))
text_df
text_df.info(verbose=True)<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 1911 entries, 0 to 1910
Data columns (total 3 columns):
 #   Column      Non-Null Count  Dtype 
---  ------      --------------  ----- 
 0   comment_id  1911 non-null   object
 1   comment     1911 non-null   object
 2   clean_text  1911 non-null   object
dtypes: object(3)
memory usage: 44.9+ KB<jupyter_text>Export cleaned data<jupyter_code>text_df.to_csv(cleaned_reddit_comment_text, index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10764662212323682, 'fraction_numerical': 0.03934669636228656, 'mean_word_length': 3.114503816793893, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '154832', 'license': 'MIT', 'n_tokens_mistral': 944, 'n_tokens_neox': 850, 'n_words': 234, 'path': 'notebooks/reddit_data_cleaning_2_jcraft.ipynb', 'repo_name': 'datakind/Mar21-vaccine-uptake', 'size': '2694'}