<jupyter_start><jupyter_text><jupyter_code>import gzip
import json
gift_data = []
with gzip.open('Gift_Cards.json.gz', 'rb') as fhand:
    for line in fhand.readlines():
        gift_data.append(line)
gift_data[0]
gift_data = []
with gzip.open('Gift_Cards.json.gz', 'rb') as fhand:
    for line in fhand.readlines():
        gift_data.append(json.loads(line))
gift_data[0]<jupyter_output><empty_output><jupyter_text>Filtering by datefor the moment, filtering just based on the reviews year.year is the last 4 characters of 'reviewTime'<jupyter_code>gift_data[0]['reviewTime'][-4:]
int(gift_data[0]['reviewTime'][-4:])<jupyter_output><empty_output><jupyter_text>Reviews after 2008<jupyter_code>rev_after_2008 = []
for rev in gift_data:
    if 'reviewTime' in rev:
        if int(rev['reviewTime'][-4:]) > 2008:
            rev_after_2008.append(rev)

rev_after_2008[0]
rev_after_2008[3]
prod_revs = []
with gzip.open('Industrial_and_Scientific_5.json.gz') as fhand:
    for line in fhand:
        prod_revs.append(json.loads(line))

prod_revs[0]
from collections import defaultdict<jupyter_output><empty_output><jupyter_text>FIltering by user activity.Removing users with less than three reviews<jupyter_code>n_reviews = defaultdict(int)
for rev in prod_revs:
    n_reviews[rev['asin']] += 1

filtered_revs = [rev for rev in prod_revs if n_reviews[rev['asin']] > 2]
len(filtered_revs)<jupyter_output><empty_output><jupyter_text>Now filtering reviews based on number of words in a review<jupyter_code>'teststring'.split()
long_revs = []
for rev in prod_revs:
    if 'reviewText' in rev:
        if len(rev['reviewText'].split()) >= 10:
            long_revs.append(rev)
long_revs[:2]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12741087083576855, 'fraction_numerical': 0.02396259497369959, 'mean_word_length': 5.584615384615384, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '524588', 'license': 'MIT', 'n_tokens_mistral': 661, 'n_tokens_neox': 641, 'n_words': 128, 'path': 'dataProcessingnVisualization/DataFilteringAndCleaning.ipynb', 'repo_name': 'Aafaaq77/appliedDataSciencewithPython', 'size': '1711'}