<jupyter_start><jupyter_text>SUMMARYThis notebook preprocesses the data and exports a tokenizer used by the deep learning models. The code is hevaliy based on [this Kaggle notebook](https://www.kaggle.com/yasufuminakama/inchi-preprocess-2).<jupyter_code>##### PACKAGES

import os
import re
import numpy as np
import pandas as pd
from tqdm.auto import tqdm
tqdm.pandas()
import torch
##### MODULES

sys.path.append('../codes')
from tokenizer import Tokenizer
from preprocessing import split_form, split_form2
##### DATA IMPORT

train = pd.read_csv('../input/train_labels.csv')
print(f'train.shape: {train.shape}')
##### PREPARE INCHI FORMULAS

train['InChI_1'] = train['InChI'].progress_apply(lambda x: x.split('/')[1])
train['InChI_text'] = train['InChI_1'].progress_apply(split_form) + ' ' + \
                        train['InChI'].apply(lambda x: '/'.join(x.split('/')[2:])).progress_apply(split_form2).values
##### FIT TOKENIZER

tokenizer = Tokenizer()
tokenizer.fit_on_texts(train['InChI_text'].values)
torch.save(tokenizer, '../input/tokenizer2.pth')
print('Saved tokenizer')
##### DISPLAY DICTIONARY

tokenizer.stoi
##### CONVERT TO TOKENS

lengths = []
tk0 = tqdm(train['InChI_text'].values, total = len(train))
for text in tk0:
    seq    = tokenizer.text_to_sequence(text)
    length = len(seq) - 2
    lengths.append(length)
train['InChI_length'] = lengths
train.to_pickle('../input/train2.pkl')
print('Saved preprocessed train.pkl')
##### CHECK SAMPLE MOLECULES

for i in range(5):
    print(train['InChI'].values[i])
    print(train['InChI_text'].values[i])
    print('-' * 50)<jupyter_output>InChI=1S/C13H20OS/c1-9(2)8-15-13-6-5-10(3)7-12(13)11(4)14/h5-7,9,11,14H,8H2,1-4H3
C 13 H 20 O S /c 1 - 9 ( 2 ) 8 - 15 - 13 - 6 - 5 - 10 ( 3 ) 7 - 12 ( 13 ) 11 ( 4 ) 14 /h 5 - 7 , 9 , 11 , 14 H , 8 H 2 , 1 - 4 H 3
--------------------------------------------------
InChI=1S/C21H30O4/c1-12(22)25-14-6-8-20(2)13(10-14)11-17(23)19-15-4-5-18(24)21(15,3)9-7-16(19)20/h13-16,19H,4-11H2,1-3H3/t13-,14+,15+,16-,19-,20+,21+/m1/s1
C 21 H 30 O 4 /c 1 - 12 ( 22 ) 25 - 14 - 6 - 8 - 20 ( 2 ) 13 ( 10 - 14 ) 11 - 17 ( 23 ) 19 - 15 - 4 - 5 - 18 ( 24 ) 21 ( 15 , 3 ) 9 - 7 - 16 ( 19 ) 20 /h 13 - 16 , 19 H , 4 - 11 H 2 , 1 - 3 H 3 /t 13 - , 14 + , 15 + , 16 - , 19 - , 20 + , 21 + /m 1 /s 1
--------------------------------------------------
InChI=1S/C24H23N5O4/c1-14-13-15(7-8-17(14)28-12-10-20(28)30)27-11-9-16-21(23(25)31)26-29(22(16)24(27)32)18-5-3-4-6-19(18)33-2/h3-8,13H,9-12H2,1-2H3,(H2,25,31)
C 24 H 23 N 5 O 4 /c 1 - 14 - 13 - 15 ( 7 - 8 - 17 ( 14 ) 28 - 12 - 10 - 20 ( 28 ) 30 ) 27 - 11 - 9 - 16 - 21 ( 23 ([...]
{'fraction_non_alphanumeric': 0.2285604900459418, 'fraction_numerical': 0.1504594180704441, 'mean_word_length': 2.7489239598278337, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '295817', 'license': 'MIT', 'n_tokens_mistral': 1379, 'n_tokens_neox': 1111, 'n_words': 249, 'path': 'notebooks/01_preprocessing_v1.ipynb', 'repo_name': 'kozodoi/BMS-Molecular-Translation', 'size': '2612'}