<jupyter_start><jupyter_text>Preprocess<jupyter_code>df = pd.read_table('data/chembl_25_chemreps.txt')
L = len(df)
print(L)
df.head()
rands = np.random.choice(L, 16, replace=False)
smiles = df.iloc[rands]['canonical_smiles'].values
mols = []
for s in smiles:
    if s is None:
        continue
    mol = MolFromSmiles(s)
    if mol is not None:
        mols.append(mol)
Draw.MolsToGridImage(mols, molsPerRow=4, subImgSize=(300,150))
lengths = list(map(len, df['canonical_smiles'].values))

plt.hist(lengths, bins=100)
plt.grid()
plt.xlabel('SMILES length')
plt.ylabel('Counts')
plt.yscale('log')
plt.hist(lengths, bins=100)
plt.grid()
plt.xlabel('SMILES length')
plt.ylabel('Counts')
plt.hist(lengths, bins=200)
plt.grid()
plt.xlim([0,200])
plt.xlabel('SMILES length')
plt.ylabel('Counts')
np.sum(np.array(lengths)<=100)/L*100
df_sub = df[np.array(lengths)<=100]
print(len(df_sub))
lengths2 = list(map(len, df_sub['canonical_smiles'].values))
print(np.max(lengths2))
df_sub.to_csv('chembl_24_100.csv', index=False)
df = pd.read_table('../data/chembl_24_chemreps.txt')
L = len(df)
print(L)
df.head()
smiles  = df['canonical_smiles'].values
to_drop = []
for i,sm in enumerate(smiles):
    if len(sm)>100:
        to_drop.append(i)
    if df['chembl_id'][i]=='CHEMBL1201364':
        to_drop.append(i)
print(len(to_drop))
df_dropped = df.drop(to_drop)
df_dropped = df_dropped.drop(['standard_inchi', 'standard_inchi_key'], axis=1)
L = len(df_dropped)
print(L)
df_dropped.head()
df_dropped.to_csv('../data/chembl_24.csv', index=False)
smiles = df['canonical_smiles'].values

# words = ['Cl', 'Br', 'At', 'Se', 'As', 'Si', 'Be', 'Ca', 'Mg', 'Li', 'Na', 'P+', 'O-', 'S-', 'Se-', 'N+', 'As+', 'Na+', 'Li+', 'H+',\
#         'K+', 'F-', 'Cl-', 'Br-', 'I-']
# for w in words:
#     cnt = 0
#     for smile in smiles:
#         if w in smile:
#             if cnt<5:
#                 print(smile)
#             cnt += 1
#     print(w, cnt/L*100, '\n')
smiles = df['canonical_smiles'].values
pro_sms = []
for sm in smiles:
    sm = ' '.join(list(sm))
    before = ['C l -', 'C l', 'O -', 'N +', 'n +', 'B r -', 'B r', 'N a +', 'N a', 'I -', 'S i']
    after = ['Cl-', 'Cl', 'O-', 'N+', 'n+', 'Br-', 'Br', 'Na+', 'Na', 'I-', 'Si']
    for b,a in zip(before, after):
        sm = sm.replace(b, a)
    pro_sms.append(sm)
df['processed_smiles'] = pro_sms
df.head()
to_drop = []
for i,sm in enumerate(smiles):
    if len(sm)>100:
        to_drop.append(i)
print(len(to_drop))
df_dropped = df.drop(to_drop)
df_dropped = df_dropped.drop(['standard_inchi', 'standard_inchi_key'], axis=1)
L = len(df_dropped)
print(L)
df_dropped.head()
df_dropped.to_csv('data/chembl_24.csv')
smiles = df_dropped['processed_smiles'].values
rands = np.random.choice(L, L, replace=False)
test_size = 10000

with open('data/sentence_test.txt', 'a') as f:
    for i in range(0, test_size):
        f.write(smiles[rands[i]]+'\n')
        
with open('data/sentence_train.txt', 'a') as f:
    for i in range(test_size, L):
        f.write(smiles[rands[i]]+'\n')<jupyter_output><empty_output><jupyter_text>Create vocabulary<jupyter_code>smiles = df_dropped['processed_smiles'].values
vocab = []
for sm in smiles:
    l = sm.split(' ')
    for w in l:
        if w not in vocab:
            vocab.append(w)

with open('data/vocab.txt', 'a') as f:
    for v in vocab:
        f.write(v+'\n')<jupyter_output><empty_output><jupyter_text>Solubility dataset<jupyter_code>df_sol = pd.read_csv('data/solubility.csv')
df_sol.head()
smiles = df_sol['SMILES'].values
pro_sms = []
for sm in smiles:
    sm = ' '.join(list(sm))
    before = ['C l -', 'C l', 'O -', 'N +', 'n +', 'B r -', 'B r', 'N a +', 'N a', 'I -', 'S i']
    after = ['Cl-', 'Cl', 'O-', 'N+', 'n+', 'Br-', 'Br', 'Na+', 'Na', 'I-', 'Si']
    for b,a in zip(before, after):
        sm = sm.replace(b, a)
    pro_sms.append(sm)

df_sol['processed_smiles'] = pro_sms
df_sol.head()
df_sol.to_csv('data/solubility.csv')
df_sol = pd.read_csv('data/solubility.csv')
df_sol.head()
N = len(df_sol)
test_size = int(N*0.25)
rands = np.random.choice(N,N, replace=False)
df_test = df_sol.iloc[rands[:test_size]]
print("Test size", len(df_test))
df_train = df_sol.iloc[rands[test_size:]]
print("Train size", len(df_train))
df_train.to_csv('data/sol_train.csv')
df_test.to_csv('data/sol_test.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2106845513413506, 'fraction_numerical': 0.01757631822386679, 'mean_word_length': 4.124407582938389, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '278865', 'license': 'MIT', 'n_tokens_mistral': 1928, 'n_tokens_neox': 1846, 'n_words': 357, 'path': 'experiments/prepare_data.ipynb', 'repo_name': 'AnshMittal1811/smiles-transformer', 'size': '4324'}