<jupyter_start><jupyter_text>Tokenization of string compounds with SmilesPE (Byte pair encoding library with built-in tokenizers)<jupyter_code>from SmilesPE.pretokenizer import atomwise_tokenizer

smi = 'CC[N+](C)(C)Cc1ccccc1Br'
toks = atomwise_tokenizer(smi)
print(toks)<jupyter_output>['C', 'C', '[N+]', '(', 'C', ')', '(', 'C', ')', 'C', 'c', '1', 'c', 'c', 'c', 'c', 'c', '1', 'Br']<jupyter_text>***example of pretrained SMILES byte-pair encoding***<jupyter_code>import requests
file_url = 'https://raw.githubusercontent.com/XinhaoLi74/SmilesPE/master/SPE_ChEMBL.txt'

r = requests.get(file_url, stream = True)

with open('/content/drive/MyDrive/Molecular Exploration/Data/BPE_codes.txt', 'wb') as file:
    for block in r.iter_content(chunk_size = 1024):
        if block:
            file.write(block)
import codecs
from SmilesPE.tokenizer import *

spe_vob= codecs.open('/content/drive/MyDrive/DATA_2040/Molecular Exploration/Data/BPE_codes.txt')
spe = SPE_Tokenizer(spe_vob)

smi = 'CC[N+](C)(C)Cc1ccccc1Br'
bpe_encoding = spe.tokenize(smi)

# should get >>> 'CC [N+](C) (C)C c1ccccc1 Br'<jupyter_output><empty_output><jupyter_text>*The output of the byte-pair encoding is a space-separated string of tokens, each token being a string. The example output below would be the input sequence to a model.*<jupyter_code>bpe_encoding.split(' ')<jupyter_output><empty_output><jupyter_text>Looking at the byte-pair encoding alphabet across the whole (~8000 large) dataset<jupyter_code># initialize the pretrained BP encoder
spe = SPE_Tokenizer(spe_vob)

# initialize empyt vocabulary set
alphabet = set()

# traverse through data adding byte-pair tokens to vocabulary
for smi in train_srp53.smiles:
    bpe_encoding = spe.tokenize(smi)
    tkns = set(bpe_encoding.split(' '))
    alphabet = alphabet.union(tkns)<jupyter_output><empty_output><jupyter_text>***The alphabet for this training set is 1096 elements -- the whole alphabet used to train this BP encoder is ~3000 ==> what do we do to prepare for getting test samples with tokens unseen in the training set?***<jupyter_code>len(alphabet)
from matplotlib import pyplot as plt
def smiles_to_token(row):
  return atomwise_tokenizer(row['smiles'])

train_srp53['tokens'] = train_srp53.apply(lambda row: smiles_to_token(row), axis=1)
train_srp53.head()
vocab = set()
for smi in train_srp53.smiles:
  tok = atomwise_tokenizer(smi)
  tokens = set(tok)
  vocab = vocab.union(tokens)

def CountFrequency(my_list):
  
    # Creating an empty dictionary 
    freq = {}
    for item in my_list:
        if item in freq:
            freq[item] += 1
        else:
            freq[item] = 1
    
    return freq

token_appears_once = {}
token_freq = {}
token_prop = {k:[] for k in vocab}
smile_lengths = []

for i, row in train_srp53.iterrows():

  token_dict = CountFrequency(row['tokens'])

  smile_lengths.append(len(row['tokens']))

  for token, count in token_dict.items():

    if token in token_appears_once.keys():
      token_appears_once[token] += 1
    else:
      token_appears_once[token] = 1

    if token in token_freq.keys():
      token_freq[token] += count
    else:
      token_freq[token] = count

  for tok in token_prop.keys():

      token_prop[tok].append(row['tokens'].count(tok) / len(row['tokens']))
    
print(token_appears_once['N'])
print(token_freq['N'])
print(len(token_prop['N']))
print(len(smile_lengths))<jupyter_output>4486
10031
8634
8634<jupyter_text>EDA Plots<jupyter_code>import plotly.express as px
from heapq import nlargest
  
def dict_to_df(d, N):
    
  # N largest values in dictionary
  # Using nlargest
  res = nlargest(N, d, key = d.get)
    
  df = pd.DataFrame(columns=['Token', 'Count'])
  df['Token'] = res

  counts = [d[token] for token in res]
  df['Count'] = counts

  return df
token_appears_df = dict_to_df(token_appears_once, 30)
fig = px.bar(token_appears_df, x='Token', y='Count')
fig.show()
token_freq_df = dict_to_df(token_freq, 30)
fig = px.bar(token_freq_df, x='Token', y='Count', log_y=True)
fig.show()
fig = px.histogram(pd.DataFrame(smile_lengths, columns=['Lengths']), x = 'Lengths')
fig.show()
prop_df = dict_to_df(token_prop, len(vocab))
prop_df['Average'] = prop_df.apply(lambda row: np.mean(row.Count), axis=1)
prop_df_sorted = prop_df.sort_values(by=['Average'], ascending=False)

# fig = px.box(prop_df_sorted.head(10), x='Token', y='Count')
# fig.show()
# prop_dict = {'C': token_prop['C'], '(':token_prop['(']}
# # prop_df = pd.DataFrame([token_prop['C'], token_prop['('], token_prop[')']], columns=['Carbon', '(', ')'])
# prop_df = pd.DataFrame(prop_dict)

# fig = px.box(prop_df, y=)
# fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14669821466982147, 'fraction_numerical': 0.015272101527210152, 'mean_word_length': 5.078431372549019, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '7770', 'license': 'MIT', 'n_tokens_mistral': 1789, 'n_tokens_neox': 1710, 'n_words': 384, 'path': 'Colab_Notebooks/EDA.ipynb', 'repo_name': 'Molecular-Exploration/toxicity-classification', 'size': '4649'}