<jupyter_start><jupyter_text>There are over 51 million reviews in this dataset. Let's take a sample distributed throughout the years.<jupyter_code>gc.get_count()
gc.collect()
import gc
gc.set_threshold(200,5,5)
j=0
selected_data=pd.DataFrame()
chunksize=1_000_000
for chunk in pd.read_json(filename, lines=True,chunksize=chunksize,nrows=100_000_000):
    selected_data=pd.DataFrame()
    dx=pd.DataFrame()
    print('Processing chunk #',j+1)
    dx=pd.DataFrame(chunk)
    dx_sample=dx.sample(frac=0.25)[['asin','overall','verified','reviewText','unixReviewTime']]
    selected_data=pd.concat([selected_data,dx_sample])
    output_file='./data/Amazon_book_reviews-Sample'+str(j+1)+'.csv'
    selected_data.to_csv(output_file,index=0)
    del dx_sample
    del selected_data
    gc.collect()
    j+=1
    if j>51:
        break
data_files=[('./data/Amazon_book_reviews-Sample'+str(i+1)+'.csv') for i in range(52)]<jupyter_output><empty_output><jupyter_text>Let's check the first 2 lines of each file to see if they are ordered in any way.<jupyter_code>df_heads=pd.DataFrame()
for file in data_files:
    file_heads=pd.read_csv(file).head(2)
    file_heads['file name']=file
    df_heads=pd.concat([df_heads,file_heads])
df_heads
df_heads.info()
df_heads['unixReviewTime']=(pd.to_datetime(df_heads['unixReviewTime'],unit='s'))
df_heads.rename(columns={'unixReviewTime':'reviewTime'},inplace=True)
df_heads<jupyter_output><empty_output><jupyter_text>Data is not ordered by date. Let's select all reviews after 2015-07-01 from all 52 data files where reviews.<jupyter_code>import gc
gc.set_threshold(150,5,5)
file_df=pd.DataFrame()
for file in data_files:
    print('processing file:',file)
    df=pd.read_csv(file)
    df['unixReviewTime']=(pd.to_datetime(df['unixReviewTime'],unit='s'))
    df.rename(columns={'unixReviewTime':'reviewTime'},inplace=True)
    df=df[df['reviewTime']>'2015-07-01'].copy(deep=True)
    file_df=pd.concat([file_df,df])
    del df
    gc.collect()

file_df
gc.collect()
file_df.head(3)
#Let's save this dataframe as a csv file
file_df.to_csv('./data/pre_processed_data.csv',index=0)
gc.collect()
del file_df
gc.collect()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14751838235294118, 'fraction_numerical': 0.028952205882352942, 'mean_word_length': 6.379661016949153, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '584885', 'license': 'Apache-2.0', 'n_tokens_mistral': 841, 'n_tokens_neox': 802, 'n_words': 134, 'path': '1_Data_Cleaning-Books_Amazon_Reviews.ipynb', 'repo_name': 'abiswas20/Classifying-Amazon-Book-Reviews', 'size': '2176'}