<jupyter_start><jupyter_text>Reading the main data-train file<jupyter_code>path = '/Users/ns5kn/Documents/insight/projects/factcc/data/processed/paired_data/data-clipped/data-train.jsonl'

with open(path) as f:
            dataset = [json.loads(line) for line in f]

print(type(dataset))

print(dataset[0].keys())

# ['id', 'claim', 'label', 'extraction_span', 'backtranslation', 'augmentation', 'augmentation_span', 'noisy', 'filepath', 'text']


print(len(dataset)) 
# 1003355
df = pd.DataFrame(dataset, columns =['id', 'claim', 'label', 'extraction_span', 'backtranslation', 'augmentation', 'augmentation_span', 'noisy', 'filepath', 'text'], dtype = float) 


# 939871 (~1/2) of the data has no augmentation applied to them. 1/2 had some sort of augmentation
# EntitySwap         151403
# NegateSentences    142225
# PronounSwap        111441
# NumberSwap          98663

df.augmentation.unique() #[None, 'EntitySwap', 'NegateSentences', 'PronounSwap', 'NumberSwap']
df['augmentation'].nunique()
df['augmentation'].value_counts()

# df['augmentation'].isnull().sum() 

df['label'].value_counts()

# REFUTES     503732
# SUPPORTS    499623<jupyter_output><empty_output><jupyter_text>create a sample<jupyter_code>sample_train = df.sample(frac=0.1, replace=True, random_state=1)

len(sample_train)
sample_train.to_csv('/Users/ns5kn/Documents/insight/projects/factcc/data/interim/sample_train_fromclipped.csv', header = True)
sample_train25 = df.sample(frac=0.25, replace=True, random_state=1)

sample_train25.to_csv('/Users/ns5kn/Documents/insight/projects/factcc/data/interim/sample_train_250_fromclipped.csv', header = True)
sample_train_01 = df.sample(frac=0.001, replace=True, random_state=1)
print(len(sample_train_01))

sample_train_01.columns

# sample_train_01.head()
sample_train_01.to_csv('/Users/ns5kn/Documents/insight/projects/factcc/data/interim/sample_train_1k_fromclipped.csv', header = True)<jupyter_output>1003<jupyter_text>Read the Sample saved<jupyter_code>import pandas as pd
train = pd.read_csv('/Users/ns5kn/Documents/insight/projects/factcc/data/interim/sample_train_fromclipped.csv')
print(train['label'].value_counts())
print(len(train))
print(train['augmentation'].value_counts())

# REFUTES     50431
# SUPPORTS    49905
# Name: label, dtype: int64

# 100336

# EntitySwap         15072
# NegateSentences    14260
# PronounSwap        11144
# NumberSwap          9955<jupyter_output>REFUTES     50431
SUPPORTS    49905
Name: label, dtype: int64
100336
EntitySwap         15072
NegateSentences    14260
PronounSwap        11144
NumberSwap          9955
Name: augmentation, dtype: int64<jupyter_text>Analyze Validation set - Manual<jupyter_code>import json
path = '/Users/ns5kn/Documents/insight/projects/factcc/data/processed/paired_data/manual/val_manual.jsonl'

with open(path) as f:
            dataset = [json.loads(line) for line in f]
print(dataset[1].keys())


# import pandas as pd

# d = pd.DataFrame(dataset, columns = ['claim', 'label', 'filepath', 'id', 'text'])<jupyter_output>dict_keys(['claim', 'label', 'filepath', 'id', 'text'])<jupyter_text>Try out the pickle<jupyter_code>import pandas as pd
d = pd.read_pickle(r'/Users/ns5kn/Documents/insight/projects/factcc/models/saved_models/29-csvfile_5batch_7epoch/history.pickle')
data = pd.DataFrame(d)

data.keys()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13801380138013802, 'fraction_numerical': 0.052505250525052505, 'mean_word_length': 5.575936883629192, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '480794', 'license': 'MIT', 'n_tokens_mistral': 1319, 'n_tokens_neox': 1197, 'n_words': 202, 'path': 'notebooks/data-exploration.ipynb', 'repo_name': 'nargesam/factCC', 'size': '3333'}