<jupyter_start><jupyter_text>Yelp Rating Prediction DatasetObtained from http://goo.gl/JyCnZq via [github](https://github.com/zhangxiangxiao/Crepe)```@misc{zhang2015characterlevel,    title={Character-level Convolutional Networks for Text Classification},    author={<NAME> and <NAME> and <NAME>},    year={2015},    eprint={1509.01626},    archivePrefix={arXiv},    primaryClass={cs.LG}}```<jupyter_code>import pandas as pd
import os
from tqdm import tqdm
import hashlib

TRAIN_DATASET_LOCATION = "../datasets/yelp/train.csv"
TEST_DATASET_LOCATION = "../datasets/yelp/test.csv"<jupyter_output><empty_output><jupyter_text>Validity check<jupyter_code>VERIFY_CHECKSUM = True
CHECKSUM = '24bbad7486bc4922501457414e082686'
assert os.path.exists(TRAIN_DATASET_LOCATION)
assert os.path.exists(TEST_DATASET_LOCATION)

def verify_checksum(file, checksum):
    with open(file, "rb") as f:
        file_hash = hashlib.md5()
        while chunk := f.read(8192):
            file_hash.update(chunk)
    assert checksum == file_hash.hexdigest()
    print("Checksum matches")
    
if VERIFY_CHECKSUM:
    verify_checksum(TRAIN_DATASET_LOCATION, '1ab7961a45755fbcaf4af11b82ee43a0')
    verify_checksum(TEST_DATASET_LOCATION, '18e419b8b7e2d321d7811f7703569a89')<jupyter_output>Checksum matches
Checksum matches<jupyter_text>Read data  For the analysis only `text` and `stars` will be used<jupyter_code>pd.read_csv(TRAIN_DATASET_LOCATION, header=None, nrows=10)
df_yelp_train = pd.read_csv(TRAIN_DATASET_LOCATION, header=None)
df_yelp_train.columns = ['target', 'text']
df_yelp_train.shape
df_yelp_train.head()
vc = df_yelp_train.target.value_counts().sort_index()
ax = vc.plot(kind='barh', title='Number of observations per stars', figsize=(10,5))
[ax.text(v, i, v) for i, v in enumerate(vc)]
ax.set_xlim(0, vc.max()*1.25)
ax.set_ylabel('Stars')
ax.set_xlabel('Count')<jupyter_output><empty_output><jupyter_text>Export File<jupyter_code>PREPROCESSED_FILE = '../datasets/preprocessed/yelp_train.csv'
df_yelp_train.to_csv(PREPROCESSED_FILE, index=False)<jupyter_output><empty_output><jupyter_text>Detect Non-English texts<jupyter_code>import fasttext
import numpy as np
from tqdm.notebook import tqdm_notebook

tqdm_notebook.pandas()

LANGUAGE_DETECTION_MODEL = "../fasttext/language_identification/lid.176.bin" #https://fasttext.cc/docs/en/language-identification.html
model = fasttext.load_model(LANGUAGE_DETECTION_MODEL)
def get_language(paragraph, k=None, return_all=False, min_certainty=0.25):
    if pd.isna(paragraph):
        return np.nan
    sentences = [line for line in paragraph.split("\n") if line != '']
    if len(sentences) < 1:
        return np.nan
    class_labels, certainties = model.predict(sentences, k)
    # sum certainties per class label and then divide by length -> avg certainty
    class_lookup, indexed_lang = np.unique(np.hstack(class_labels), return_inverse=True)
    summed_certainties = np.bincount(indexed_lang, weights=np.hstack(certainties))/len(sentences)
    if return_all:
        return dict(zip(class_lookup, summed_certainties))
    
    # get most likely lang
    lang_id = np.argmax(summed_certainties)
    if summed_certainties[lang_id] < min_certainty:
        return np.nan
    return class_lookup[lang_id]        
df_yelp_train['lang'] = df_yelp_train.text.progress_apply(lambda review: get_language(review, k=5))
vc = df_yelp_train.lang.value_counts().sort_index()
ax = vc.plot(kind='barh', title='Number of observations per language', figsize=(10,5))
[ax.text(v, i, v) for i, v in enumerate(vc)]
ax.set_xlim(0, vc.max()*1.25)
ax.set_ylabel('Language')
ax.set_xlabel('Count')
vc.nlargest(5)
f"{vc['__label__en']/df_yelp_train.shape[0]*100:.4f}% of samples are english"<jupyter_output><empty_output><jupyter_text>Test Set analysis<jupyter_code>df_yelp_test = pd.read_csv(TEST_DATASET_LOCATION, header=None)
df_yelp_test.columns = ['target', 'text']
df_yelp_test.shape
df_yelp_test.head()
vc = df_yelp_test.target.value_counts().sort_index()
ax = vc.plot(kind='barh', title='Number of observations per stars', figsize=(10,5))
[ax.text(v, i, v) for i, v in enumerate(vc)]
ax.set_xlim(0, vc.max()*1.25)
ax.set_ylabel('Stars')
ax.set_xlabel('Count')<jupyter_output><empty_output><jupyter_text>Export File<jupyter_code>PREPROCESSED_FILE = '../datasets/preprocessed/yelp_test.csv'
df_yelp_test.to_csv(PREPROCESSED_FILE, index=False)
df_yelp_test['lang'] = df_yelp_test.text.progress_apply(lambda review: get_language(review, k=5))
vc = df_yelp_test.lang.value_counts().sort_index()
ax = vc.plot(kind='barh', title='Number of observations per language', figsize=(10,5))
[ax.text(v, i, v) for i, v in enumerate(vc)]
ax.set_xlim(0, vc.max()*1.25)
ax.set_ylabel('Language')
ax.set_xlabel('Count')
vc.nlargest(5)
f"{vc['__label__en']/df_yelp_test.shape[0]*100:.4f}% of samples are english"<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13468291675273703, 'fraction_numerical': 0.028919644701507954, 'mean_word_length': 6.460708782742681, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 32, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '292823', 'license': 'Apache-2.0', 'n_tokens_mistral': 1970, 'n_tokens_neox': 1849, 'n_words': 302, 'path': 'preprocessing/yelp.ipynb', 'repo_name': 'freecraver/zimp_experiments', 'size': '4854'}