<jupyter_start><jupyter_text>Balancing<jupyter_code>df_non = df[df['class'] == 0]
df_spoiler = df[df['class'] == 1]
print("nonspoiler:", len(df_non))
print("spoiler:", len(df_spoiler))
df_non = df_non.sample(frac =.05)
print("weight-decreased nonspoiler:", len(df_non))
print("balanced total:", len(df_non)+len(df_spoiler))<jupyter_output>nonspoiler: 17102931
spoiler: 569724
weight-decreased nonspoiler: 855147
balanced total: 1424871<jupyter_text>train test split<jupyter_code># 70/10/20 train/val/test
df_train_non = df_non.sample(frac = .70)
df_non = df_non.drop(df_train_non.index)
df_train_spoiler = df_spoiler.sample(frac = .70)
df_spoiler = df_spoiler.drop(df_train_spoiler.index)

# use 33% of the remaining for validation (which is 10% of whole dataset)
df_dev_non = df_non.sample(frac = .333)
df_non = df_non.drop(df_dev_non.index)
df_dev_spoiler = df_spoiler.sample(frac = .333)
df_spoiler = df_spoiler.drop(df_dev_spoiler.index)

# use all of the remaining for test (which is 20% of whole dataset)
df_test_non = df_non
df_test_spoiler = df_spoiler
# merge and shuffle

df_train = pd.concat([df_train_non, df_train_spoiler])
df_train = df_train.sample(frac=1).reset_index(drop=True)

df_dev = pd.concat([df_dev_non, df_dev_spoiler])
df_dev = df_dev.sample(frac=1).reset_index(drop=True)

df_test = pd.concat([df_test_non, df_test_spoiler])
df_test = df_test.sample(frac=1).reset_index(drop=True)

print("train:",len(df_train))
print("dev:",len(df_dev))
print("test:",len(df_test))
print("all:",len(df_train)+len(df_dev)+len(df_test))
df_train.to_csv('csv_datasets/goodreads/train.csv', index=False)
df_dev.to_csv('csv_datasets/goodreads/dev.csv', index=False)
df_test.to_csv('csv_datasets/goodreads/test.csv', index=False)<jupyter_output><empty_output><jupyter_text>sample to try<jupyter_code>df_train_sample = df_train.sample(7000)
df_dev_sample = df_dev.sample(1000)
df_test_sample = df_test.sample(2000)
df_train_sample.to_csv('csv_datasets/goodreads/sample/train.csv', index=False)
df_dev_sample.to_csv('csv_datasets/goodreads/sample/dev.csv', index=False)
df_test_sample.to_csv('csv_datasets/goodreads/sample/test.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1427260211106012, 'fraction_numerical': 0.031206975676916018, 'mean_word_length': 8.77578475336323, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '469177', 'license': 'MIT', 'n_tokens_mistral': 952, 'n_tokens_neox': 891, 'n_words': 123, 'path': 'preprocess.ipynb', 'repo_name': 'smtnkc/bert-spoiler', 'size': '2179'}