<jupyter_start><jupyter_text>Reddit Preprocessing in StagesThis notebook is essentially the `reddit_preprocessor.py` file in the data package, but split into disjoint steps. This is useful for both development and visualization, e.g. you can save the data at each stage of the processing pipeline and resume later, rather than requiring a restart from beginning.<jupyter_code>import os
import sys
sys.path.append('..')
from imp import reload

from data import reddit_preprocessor, DataHelper
from data.reddit_preprocessor import *

import json
import pandas as pd
from pprint import pprint
data_helper = DataHelper()<jupyter_output>Using logfile: /tmp/data_helper_1xuld5d
Hi, I'm a DataHelper. For now, I help with the reddit dataset.
At any prompt, press ENTER if you want the default value.
Username (default='brandon'): 
Hello, brandon, I've set your data root to /home/brandon/Datasets/reddit
Years to process (default='2007,2008,2009'): 
These are the files I found:
['/home/brandon/Datasets/reddit/raw_data/2007/RC_2007-11',
 '/home/brandon/Datasets/reddit/raw_data/2007/RC_2007-12',
 '/home/brandon/Datasets/reddit/raw_data/2007/RC_2007-10',
 '/home/brandon/Datasets/reddit/raw_data/2008/RC_2008-12',
 '/home/brandon/Datasets/reddit/raw_data/2008/RC_2008-07',
 '/home/brandon/Datasets/reddit/raw_data/2008/RC_2008-08',
 '/home/brandon/Datasets/reddit/raw_data/2008/RC_2008-03',
 '/home/brandon/Datasets/reddit/raw_data/2008/RC_2008-06',
 '/home/brandon/Datasets/reddit/raw_data/2008/RC_2008-05',
 '/home/brandon/Datasets/reddit/raw_data/2008/RC_2008-09',
 '/home/brandon/Datasets/reddit/raw_data/2008/RC_2008-01',
 '/home/bra[...]<jupyter_text>Playing with the Raw DataUse data helper to load the uncompressed data into a pandas DataFrame.<jupyter_code>data = {}
years = list(range(2008, 2010 + 1))
for year in range(2008, 2010 + 1):
    print('Loading year:', year)
    data[str(year)] = data_helper.load_random(year=year)
print('Done!')<jupyter_output>Loading year: 2008
Returning data from file:
 /home/brandon/Datasets/reddit/raw_data/2008/RC_2008-11
Loading year: 2009
Returning data from file:
 /home/brandon/Datasets/reddit/raw_data/2009/RC_2009-08
Loading year: 2010
Returning data from file:
 /home/brandon/Datasets/reddit/raw_data/2010/RC_2010-03
Done!<jupyter_text>Remove entries with the comment as "deleted".<jupyter_code>years = list(range(2008, 2010 + 1))
for year in years:
    df = data[str(year)]
    data[str(year)] = df.loc[df.body != '[deleted]'].reset_index(drop=True)
jtplot.style('onedork', ticks=True, fscale=1.5)
jtplot.figsize(x=11., y=8.)

# Plot the inner 90 of data for 'score'. 
years = ['2008', '2009', '2010']
score_ranges = [tuple(data[y].score.quantile([0.05, .95])) for y in years]
print(score_ranges)
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline

from jupyterthemes import jtplot
jtplot.style('monokai')
# set "context" (paper, notebook, talk, or poster)
# & font scale (scalar applied to labels, legend, etc.)
jtplot.style(ticks=True, grid=False)
jtplot.figsize(x=10, y=8)
# [Optional] cut down size of data.
percent_keep = 0.9
for year in range(2008, 2010 + 1):
    print('Loading year:', year)
    num_keep = int(percent_keep * len(data[str(year)].index))
    data[str(year)] = data[str(year)].loc[:num_keep]
df_small.columns<jupyter_output><empty_output><jupyter_text>Visualization: Comment Scores<jupyter_code>jtplot.style('onedork', ticks=True, fscale=1.5)
jtplot.figsize(x=11., y=8.)

# Plot the inner 90 of data for 'score'. 
years = ['2008', '2009', '2010']
score_ranges = [tuple(data[y].score.quantile([0.05, .95])) for y in years]
print(score_ranges)

# Pick the widest range to fit all data in plot.
from operator import itemgetter
score_range = [min(score_ranges, key=itemgetter(0))[0],
               max(score_ranges, key=itemgetter(1))[1]]
print('sr:', score_range)
num_bins = int(score_range[1] - score_range[0] + 1)
print('nb:', num_bins)

score_data = [data[y].score for y in years]
plt.hist(score_data,
         bins=num_bins,
         normed=True,
         align='left',
         label=years,
         range=score_range)

plt.title('Comment Scores (Normalized)')
plt.xlabel('Score')
plt.ylabel('Counts')
plt.legend()
plt.show()<jupyter_output>[(-1.0, 15.0), (-1.0, 15.0), (0.0, 13.0)]
sr: [-1.0, 15.0]
nb: 17<jupyter_text>Visualization: Word Distributions<jupyter_code>from nltk import word_tokenize
from collections import Counter
from itertools import chain

jtplot.style('onedork', ticks=True, fscale=1.5)
jtplot.figsize(x=11., y=8.)

#$word_tokenize(data['2008'].loc[0, 'body'])
pd.set_option('display.max_colwidth', 1000)
#pd.DataFrame(data['2008']['body']).head()
from nltk import FreqDist

wt = Counter(chain.from_iterable(data['2008']['body']))
tokenized = parallel_map_list(fn=DataHelper.word_tokenizer, 
                              iterable=data['2008'].body.values)
print('DONE')
data_helper.set_word_freq(Counter(chain.from_iterable(tokenized)))
most_comm = data_helper.word_freq.most_common(20)
words, counts = zip(*most_comm)

x_labels = words
counts_series = pd.Series.from_array(counts)

plt.figure(figsize=(16, 10))
ax = counts_series.plot(kind='bar')
ax.set_title('Most Common Words 2008')
ax.set_ylabel('Counts')
ax.set_xlabel('Words')
ax.set_xticklabels(x_labels)<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>PROJECT_ROOT = '/home/brandon/Documents/DeepChatModels'
REDDIT_DIR = '/home/brandon/Datasets/reddit'
SOURCE_DIR = os.path.join(REDDIT_DIR, 'raw_data')
TARGET_DIR = os.path.join(REDDIT_DIR, 'raw_data_columns_removed')
YEAR = '2009'
TARGET_FILE = os.path.join(TARGET_DIR, YEAR + '_cols_removed.json')
data_helper = DataHelper()
data_helper.file_counter = 9
print(data_helper.file_paths[data_helper.file_counter:])<jupyter_output><empty_output><jupyter_text>Load 2007 Data in DF<jupyter_code>df = data_helper.safe_load()
print("df has {} rows.".format(len(df.index)))
print("df has columns:\n", df.columns)
df.body.head()
df = remove_extra_columns(df)
print("df has {} rows.".format(len(df.index)))
print("df has columns:\n", df.columns)
df.head()
DataHelper.df_to_json(df, orient='records', lines=False, target_file=TARGET_FILE)<jupyter_output><empty_output><jupyter_text>In One Function<jupyter_code>def remove_extra_columns_and_save()
    print("Removing extra columns . . . ")
    df = remove_extra_columns(df)
    print("Saving to {} . . . ".format(TARGET_FILE))
    DataHelper.df_to_json(
        df, 
        orient='records', 
        lines=False, 
        target_file=TARGET_FILE)
    print("Done!")<jupyter_output><empty_output><jupyter_text>Loading back later<jupyter_code>df_loaded = pd.read_json(target_file)
len(df_loaded)
file_names = []
for y in ['2007', '2008', '2009']:
    file_names.append(y + '_cols_removed.json')
file_names = file_names[:-1]
#file_names[-1] = '2009_cols_removed_part_1.json'
#file_names.append('2009_cols_removed_part_2.json')
file_names = [os.path.join(TARGET_DIR, fname) for fname in file_names]
pprint(file_names)
list_ = []
for file_path in file_names:
    print('file:', file_path)
    sys.stdout.flush()
    list_.append(pd.read_json(file_path))
df = pd.concat(list_).reset_index()
del list_
list_ = None
TARGET_FILE = os.path.join(TARGET_DIR, '2007-8_cols_removed.json')
DataHelper.df_to_json(df, orient='records', lines=False, target_file=TARGET_FILE)<jupyter_output><empty_output><jupyter_text>Stage 2: Regex, Long Comments, and Contractions  Regex<jupyter_code>TARGET_FILE = os.path.join(TARGET_DIR, YEAR+'_cols_removed_part_1.json')
df = pd.read_json(TARGET_FILE)
print('DataFrame loaded.')
from data.reddit_preprocessor import regex_replacements
df = regex_replacements(df)
print('Regex replacements done.')
TARGET_FILE = os.path.join(TARGET_DIR, '2007-8_regex_replaced.json')
DataHelper.df_to_json(df, orient='records', lines=False, target_file=TARGET_FILE)<jupyter_output><empty_output><jupyter_text>Long Comments<jupyter_code>TARGET_FILE = os.path.join(TARGET_DIR, '2007-8_regex_replaced.json')
df = pd.read_json(TARGET_FILE)
print('DataFrame loaded from', TARGET_FILE)
reload(data)
from data.reddit_preprocessor import remove_large_comments, expand_contractions
df = remove_large_comments(df=df, max_len=20)
df = expand_contractions(df)
print('Remove large comments and contractions done.')

TARGET_FILE = os.path.join(TARGET_DIR, YEAR+'_large_comments_contractions.json')
DataHelper.df_to_json(df, orient='records', lines=False, target_file=TARGET_FILE)

print('done')<jupyter_output><empty_output><jupyter_text>Stage 3: Parallel Calls<jupyter_code>from data.reddit_preprocessor import parallel_map_list
from data.reddit_preprocessor import data_helper
TARGET_FILE = os.path.join(TARGET_DIR, '2007-8_large_comments_contractions.json')
df = pd.read_json(TARGET_FILE)
print('DataFrame loaded from', TARGET_FILE)
from collections import Counter
from itertools import chain
sentences = parallel_map_list(fn=DataHelper.word_tokenizer, iterable=df.body.values)
data_helper.set_word_freq(Counter(chain.from_iterable(sentences)))
from data.reddit_preprocessor import sentence_score
df['score'] = parallel_map_list(fn=sentence_score, iterable=sentences)
del sentences
import matplotlib.pyplot as plt
from jupyterthemes import jtplot
jtplot.style('onedork')
jtplot.style(ticks=True, grid=False)
jtplot.figsize(aspect=1.2)
plt.hist(df['score'].values, bins=100,
        range=(0., df['score'].quantile(0.7)))
plt.show()<jupyter_output><empty_output><jupyter_text>Final Stage<jupyter_code># Keep the desired percentage of lowest-scored sentences. (low == good)
from data.reddit_preprocessor import children_dict
keep_best_percent = 0.7
df = df.loc[df['score'] < df['score'].quantile(keep_best_percent)]

print('Prepping for the grand finale.')
comments_dict = pd.Series(df.body.values, index=df.name).to_dict()
root_to_children = children_dict(df)
data_helper.generate_files(
            from_file_path="from_file.txt",
            to_file_path="to_file.txt",
            root_to_children=root_to_children,
            comments_dict=comments_dict)<jupyter_output>Prepping for the grand finale.
Time to run children_dict: 4.701 seconds.
Final processed file has 306164 samples total.<jupyter_text>Exploration<jupyter_code>df.style.set_properties(subset=['text'], **{'width': '800px'})
def get_comments(df, n=10):
    i = 0
    while i < len(df.index):
        yield df.loc[i:i+n]['body']
        i += n
gen = get_comments(df, 5)
comments = next(gen)
for comment in comments:
    print(comment)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13532763532763534, 'fraction_numerical': 0.038461538461538464, 'mean_word_length': 6.0394385026737964, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '303019', 'license': 'MIT', 'n_tokens_mistral': 4060, 'n_tokens_neox': 3760, 'n_words': 738, 'path': 'notebooks/RedditPipelineAndVisualization.ipynb', 'repo_name': 'shreyarora12/DeepChatModels', 'size': '10530'}