<jupyter_start><jupyter_text>remove duplicates<jupyter_code>from rdkit import Chem
toCanonical = lambda x:Chem.MolToSmiles(Chem.MolFromSmiles(x), canonical=True)
df.smiles = df.smiles.apply(toCanonical)
df_nodup = df.drop_duplicates('smiles')
df.shape, df_nodup.shape
dfo = pd.read_csv('../data/bace.csv')
dfo.smiles = dfo.smiles.apply(toCanonical)<jupyter_output><empty_output><jupyter_text>remove the same data with the molnet bace dataset<jupyter_code>df_nodup_exclude_bace = df_nodup[~df_nodup.smiles.isin(dfo.smiles)]
dfn = df_nodup_exclude_bace.reset_index(drop=True)
dfn.set_index('ID').to_csv('../data/bace_chembl.csv')
dfn.groupby(['Assay', 'Class']).size()
len(dfn)
import pandas as pd
a = pd.read_csv('../data/bace_chembl.csv')
a.Class.value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1504424778761062, 'fraction_numerical': 0.0, 'mean_word_length': 8.10344827586207, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '468648', 'license': 'MIT', 'n_tokens_mistral': 338, 'n_tokens_neox': 327, 'n_words': 41, 'path': 'paper/07_BACE_HPs/chembl_data_preprocessing/preprocessing.ipynb', 'repo_name': 'riversdark/bidd-molmap', 'size': '791'}