<jupyter_start><jupyter_text>Setting up<jupyter_code>import pandas as pd
from sklearn.model_selection import train_test_split

# enable multiple outputs per cell
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"<jupyter_output><empty_output><jupyter_text>Preparing dataset into dataframe<jupyter_code># header for dataset
headernames = ['id', 'label', 'statement', 'subject', 'speaker', 'job', 'state', 'party',
                   'barely_true', 'false', 'half_true', 'mostly_true', 'pants_on_fire', 'venue']

df_train = pd.read_csv('./datasets/liar_dataset/train.tsv', sep='\t', names=headernames)
df_test = pd.read_csv('./datasets/liar_dataset/test.tsv', sep='\t', names=headernames)
df_valid = pd.read_csv('./datasets/liar_dataset/valid.tsv', sep='\t', names=headernames)
# grouping labels into true or fake<jupyter_output><empty_output><jupyter_text>Having a look<jupyter_code># remove duplicate data entries
df_train.drop_duplicates(subset ="statement", 
                     keep = False, inplace = True)
df_test.drop_duplicates(subset ="statement", 
                     keep = False, inplace = True)
df_valid.drop_duplicates(subset ="statement", 
                     keep = False, inplace = True)
# we are only doing text-content analysis
# creating new dataframe with only statement and label
statement_train = df_train["statement"]
label_train = df_train["label"]
train_cleaned = pd.concat([statement_train, label_train], axis=1)
statement_test = df_test["statement"]
label_test = df_test["label"]
test_cleaned = pd.concat([statement_test, label_test], axis=1)
statement_valid = df_valid["statement"]
label_valid = df_valid["label"]
valid_cleaned = pd.concat([statement_valid, label_valid], axis=1)
# change label string to numerical values
truth_val = {'false':0.,'half-true':0.5,'mostly-true':0.75,'true':1.
         ,'pants-fire':-0.25,'barely-true':0.25}

train_cleaned = train_cleaned.replace({"label":truth_val})
test_cleaned = test_cleaned.replace({"label":truth_val})
valid_cleaned = valid_cleaned.replace({"label":truth_val})
train_cleaned.to_csv('./datasets/train_cleaned.csv', sep=',', encoding='utf-8', index=False)
test_cleaned.to_csv('./datasets/test_cleaned.csv', sep=',', encoding='utf-8', index=False)
valid_cleaned.to_csv('./datasets/valid_cleaned.csv', sep=',', encoding='utf-8', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14653383146533833, 'fraction_numerical': 0.0078870900788709, 'mean_word_length': 6.1301775147929, 'pattern_counts': {'":': 3, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '356466', 'license': 'MIT', 'n_tokens_mistral': 822, 'n_tokens_neox': 798, 'n_words': 148, 'path': 'data-cleaning.ipynb', 'repo_name': 'SasungKim/EB01', 'size': '2409'}