<jupyter_start><jupyter_text>Filter Original Data<jupyter_code>DATASET = 'wsc-spacy'
wd = os.path.dirname(os.path.abspath('__file__'))

f_name = 'Bert WSC Results.csv'

results = pd.read_csv(os.path.join(wd, f_name))
print(list(results.columns.values))
print(f"\n {results.shape}")

framings = results['framing'].unique()
print(framings)
print(results.shape[0])
print(len(results['exp_name'].unique()))
print(list(results['learning_rate'].unique()))
print(list(results['max_epochs'].unique()))
print(list(results['batch_size'].unique()))
lr_c = list(results['learning_rate'].unique())[:3]
bs_c = list(results['batch_size'].unique())
ep_c = list(results['max_epochs'].unique())[:3]
data = 'wsc-spacy'

lr_c.append(0)
ep_c.append(0)

hp_space = pd.DataFrame({'learning_rate':lr_c, 'batch_size':bs_c, 'max_epochs':ep_c, 'dataset':data})

print(lr_c)
print(bs_c)
print(ep_c)
lr = 'learning_rate'
bs = 'batch_size'
ep = 'max_epochs'
d = 'dataset'

keep = []

frames = {key:0 for key in framings}

for label, row in results.iterrows():
    if (row['dataset'] == data and
        row['learning_rate'] in lr_c and
        row['batch_size'] in bs_c and
        row['max_epochs'] in ep_c):
        keep.append(label)
        frames[row['framing']] += 1

print(f"Kept {len(keep)}")
print(frames)
filtered_results = results.loc[keep,:].reset_index()
print(filtered_results.shape[0])
print(len(filtered_results['exp_name'].unique()))<jupyter_output>420
420<jupyter_text>Save<jupyter_code>wd = os.path.dirname('__file__')
out_results = os.path.join(wd, f"{DATASET}_RESULTS_BERT.csv")
framing_counts = os.path.join(wd,f"{DATASET}_FRAMING_COUNTS_BERT.json")
hp_name = os.path.join(wd,f"{DATASET}_HP_SPACE_BERT.json")

filtered_results.to_csv(out_results)
hp_space.to_json(hp_name)

with open(framing_counts, 'w') as f:
    f.write(json.dumps(frames))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18186695278969958, 'fraction_numerical': 0.0075107296137339056, 'mean_word_length': 6.9361702127659575, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '25480', 'license': 'MIT', 'n_tokens_mistral': 750, 'n_tokens_neox': 747, 'n_words': 104, 'path': 'analysis/wsc-spacy-filter-data-BERT.ipynb', 'repo_name': 'HaokunLiu/wsc-trick', 'size': '1864'}