<jupyter_start><jupyter_text>Split original dataset into training and testing datasets<jupyter_code>import numpy as np
import pandas as pd
from src.config import raw_data_path, training_dataset_path, testing_dataset_path
%cd ..
import src.data.notebook_utils as utils

df1 = utils.read_data(raw_data_path, organic_key=True)
len(df1)
from src.config import training_dataset_path, data_types_path
import json

with open(data_types_path) as json_file:

    dtypes = json.load(json_file)
        
    df = pd.read_csv("data/raw/full-perovskite-data.csv", header=0)
len(df)
df[["_raw_ExpVer", "_raw_reagent_0_chemicals_0_InChIKey"]].value_counts()

df_2 = df.loc[df['_raw_reagent_0_chemicals_0_InChIKey'] == '<KEY>'] 
values2 = list(df_2['_rxn_organic-inchikey'].unique())
values1 = list(df1['_rxn_organic-inchikey'].unique())
overlap = [val for val in values2 if val in values1]
df_groupedby_inchi = df_2.groupby(by=['_rxn_organic-inchikey', '_out_crystalscore'])
ax = df_groupedby_inchi.size().unstack().plot.barh(stacked=True, figsize=(10,15), label='Crystal Score')

columns = utils.get_columns(df.columns)
columns['all'] = columns['_rxn_'].union(columns['_feat_'], {'_out_crystalscore'}) 
only_amine = df1.loc[df1['_rxn_organic-inchikey'] == '<KEY>'] # EtNH3I
only_amine[['_out_crystalscore']].value_counts()
only_amine.to_csv("data/preproccess/only_one_amine.csv", index=None)
only_amine.columns
# filter very imbalanced amines
df.drop(df.loc[df['_rxn_organic-inchikey'] == 'XFYICZOIWSBQSK-UHFFFAOYSA-N'].index, inplace = True) #EtNH3I
df.drop(df.loc[df['_rxn_organic-inchikey'] == 'JMXLWMIFDJCGBV-UHFFFAOYSA-N'].index, inplace = True) #MENH2I
df.drop(df.loc[df['_rxn_organic-inchikey'] == 'GGYGJCFIYJVWIP-UHFFFAOYSA-N'].index, inplace = True) # AcNH3I

df['_out_crystalscore'] = df['_out_crystalscore'].astype(int)
df.drop(df.loc[df['_out_crystalscore'] == 0].index, inplace = True)<jupyter_output><empty_output><jupyter_text>Select only required columns<jupyter_code>df = df[list(columns['all'])]<jupyter_output><empty_output><jupyter_text>Raw data split into balanced training and testing sets<jupyter_code>train = df.groupby('_out_crystalscore').sample(frac=0.95, random_state=200) 
test = df.drop(train.index)
print("Train dataset size: {} \n Test dataset size: {}".format(len(train), len(test)))
train.head()
train.to_csv(training_dataset_path, index=None)
test.to_csv(testing_dataset_path, index=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14883148831488316, 'fraction_numerical': 0.013120131201312012, 'mean_word_length': 8.104477611940299, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '256691', 'license': 'MIT', 'n_tokens_mistral': 988, 'n_tokens_neox': 945, 'n_words': 142, 'path': 'notebooks/0.0-mbto-split_raw_data.ipynb', 'repo_name': 'beluticona/licentiate-thesis-repo', 'size': '2483'}