<jupyter_start><jupyter_text>Data cleaning for yelp dataset<jupyter_code>data = []
with open('../data/yelp-dataset/yelp_academic_dataset_review.json',encoding="utf8") as f:
    for line in f:
        data.append(json.loads(line))
DF = pd.DataFrame(data)
DF_drop = DF[DF['stars']!= 3.0]
DF_drop['stars'].value_counts()
positive = []

for i in DF_drop.stars:
    if i < 3.0:
        positive.append(0)
    else:
        positive.append(1)
DF_drop['positive'] = positive
DF_pos = DF_drop[DF_drop['positive']==1]
DF_neg = DF_drop[DF_drop['positive']==0]
DF_pos_sample = DF_pos.sample(n=50000,random_state=123)
DF_neg_sample = DF_neg.sample(n=50000,random_state=123)
DF_pos_sample['stars'].value_counts()
DF_neg_sample['stars'].value_counts() 
yelp = pd.concat([DF_pos_sample,DF_neg_sample])
yelp.to_csv('../data/yelp-dataset/df_yelp.csv',index=False)
yelp = pd.read_csv('../data/yelp-dataset/df_yelp.csv')
# Code to detect languages and then remove those not labelled as English. This removed about 500 examples but did
# not clean all of the text of non english words. This might not be an issue though but not sure.

yelp.text.isna().value_counts()

count = 0 #check for empty strings
for i in range(len(yelp.text)):
    if not yelp.text[i].strip():
        count +=1
    else:
        continue

print(count)

DetectorFactory.seed = 0
final = []
for i in range(len(yelp.text)):
    try:
        language = detect(yelp.text[i])
    except:
        language = "error"
        print("This row throws an error:", yelp.text[i])
    final.append(language)


yelp['lang']= final
yelp = yelp[yelp['lang']=='en']

yelp.lang.value_counts()
yelp = yelp.drop(['lang','useful','stars','review_id','user_id','business_id','cool','funny','date'],axis=1)
yelp.to_csv('../data/yelp-dataset/yelp2.csv',index=False)<jupyter_output><empty_output><jupyter_text>Data cleaning for coursera dataset<jupyter_code>DF = pd.read_csv('../data/coursera/reviews_coursera.csv')

DF = DF.rename(columns={'Review':'text','Label':'stars'})

DF_drop = DF[DF['stars']!= 3.0]

DF_drop['stars'].value_counts()
DF_drop = DF_drop.drop(['Id'],axis=1)
positive = []

for i in DF_drop.stars:
    if i < 3.0:
        positive.append(0)
    else:
        positive.append(1)
DF_drop['positive'] = positive

DF_pos = DF_drop[DF_drop['positive']==1]
DF_neg = DF_drop[DF_drop['positive']==0]

DF_pos_sample = DF_pos.sample(n=4000,random_state=123)
DF_neg_sample = DF_neg.sample(n=4000,random_state=123)

DF_pos_sample['stars'].value_counts()
DF_neg_sample['stars'].value_counts()
coursera = pd.concat([DF_pos_sample,DF_neg_sample])

coursera = coursera.drop(['stars'],axis=1)

coursera = coursera.reset_index(drop=True)
coursera = coursera.rename(columns={'Review':'text','positive':'pos_class'})
coursera
coursera.to_csv('../data/coursera_clean.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16684340756451044, 'fraction_numerical': 0.02014846235418876, 'mean_word_length': 5.302895322939866, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '100207', 'license': 'MIT', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1066, 'n_words': 180, 'path': 'code/DataCleaning_yelp_coursera.ipynb', 'repo_name': 'halmga/Generalized_Sentiment_model', 'size': '2829'}