<jupyter_start><jupyter_text>UPDATED SPLITS WITH SPECIES: Preprocess uniprot data (SPs that are only experimentally verified and verified by sequence analysis) and split into train/val/test with tokens for the species.First, go through and split the sequences into the signal peptide and the remainder of the sequence. Discard sequences where the signal peptide does not start at the first position. Then, discard sequenceswhere the signal peptide is not between 10 and 70 amino acids, inclusive. Also discard sequences where the remaining sequence is not strictly longer than the signal peptide. In one training dataset, keep the first 100 amino acids of the mature protein. In another training dataset, only keep the first 95, 100, and 105 amino acids of the mature protein in the training dataset to vary the length of the protein sequences. This way, we get "more" training data if for each one.Remove examples where the SP is the same and the protein sequences are > 0.5 the same.For each example, also save the organism. All organisms with fewer than 5 examples get lumped together as token 0: 'AAUnknown' There are a total of 754 species tokens. There are a total of 32263 examples. Finally, shuffle the signal peptide/mature protein pairs and set aside 20% each as test and validation sets. The split is 19359/6452/6452. Minimal SP Length: 10 AAMaximal SP Length: 70 AA Defaults from SignalP http://www.cbs.dtu.dk/services/SignalP/instructions.phplimits Minimal Protein Length: Longer than Signal PeptideMaximal Protein Length: truncated to 70 -> according the SignalP’s SI (below)https://images.nature.com/full/nature-assets/nmeth/journal/v8/n10/extref/nmeth.1701-S1.pdf<jupyter_code>%matplotlib inline
import pickle
import random
from collections import Counter

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('whitegrid')

import csv
# read in datasets from csv
dataset_75 = []

filename = "dataset_2.csv"
with open(filename, "r") as f:
    reader = csv.reader(f, delimiter="\t")
    index = 0 # for appending to lists of similarities within the list "sim"
    for j, line in enumerate(reader): # reads in each row of the csv
        dataset_75.append(line[0])
# load in prot sequences from dataset
df = pd.read_excel('../dataset.xls')
si_c = df['Signal peptides'].values
pr_c = df['Prot Sequences'].values
# include triplets of all lengths
triplets = []

for si, pr in zip(si_c, pr_c):
    if pr in dataset_75:
        triplets.append((si, pr))
# Remove exact duplicates
triplets = list(set(triplets))
len(triplets)
random.seed(a=1)
random.shuffle(triplets)
L = len(triplets) // 5
test = triplets[-L:]
val = triplets[-2 * L:-L]
train = triplets[:-2*L]
len(train), len(val), len(test)#check tokens
#check similarity
# Ensure prot seq length of val and test are 100 aa
# dataset where training has prot seqs of length 100

train_len = [(si, pr[:100]) for si, pr in train]
val_len = [(si, pr[:100]) for si, pr in val]
test_len = [(si, pr[:100]) for si, pr in test]

val = val_len
test = test_len
# number of pairs with prot seq of length less than 100
length = 0

for t in train:
    sp, pr = t
    if len(pr) < 100:
        length += 1

length
with open('../../6-11_data/train_75.pkl', 'wb') as f:
    pickle.dump(train_len, f)
with open('../../6-11_data/validate_75.pkl', 'wb') as f:
    pickle.dump(val, f)
with open('../../6-11_data/test_75.pkl', 'wb') as f:
    pickle.dump(test, f)
# Python code to remove duplicate elements
def Remove(duplicate):
    final_list = []
    dup = []
    for num in duplicate:
        if num not in final_list:
            final_list.append(num)
        else:
            dup.append(num)
    return dup
# vary prot length in training set
maximum = 105
leng1 = 0
leng2 = 0
train_vlen = []
train_vlen1 = []
train_vlen2 = []

####### keep train with og lengths, len, len-5, len-10 (smaller of 105 or actual protein length for len)
for t in train:
    si, pr = t
    if len(pr) < maximum:
        leng1 = len(pr) - 10
        leng2 = len(pr) - 5
        train_vlen.append((si, pr[:leng1]))
        train_vlen1.append((si, pr[:leng2]))
        train_vlen2.append((si, pr))
    else:
        train_vlen.append((si, pr[:95]))
        train_vlen1.append((si, pr[:100]))
        train_vlen2.append((si, pr[:105]))

print(len(train_vlen))
print(len(train_vlen1))
print(len(train_vlen2))

train = []
train = train_vlen + train_vlen1 + train_vlen2

print(len(train))
# Remove exact duplicates
train = list(set(train))
print(len(train))
# dump data with sp, prot, and species in *_species_augmented.pkl files (training dataset with varied prot seq lengths of 
# 95, 100, and 105)

with open('../../6-11_data/train_augmented_75.pkl', 'wb') as f:
    pickle.dump(train, f)
with open('../../6-11_data/train_75.pkl', 'rb') as f:
    t = pickle.load(f)
t[0]
# remove species from datasets to be dumped into train.pkl, test.pkl, and validate.pkl
# has varied lengths of 95, 100, and 105

train_nosp = [(si, pr) for si, pr in train]

with open('../../6-11_data/train_augmented_75.pkl', 'wb') as f:
    pickle.dump(train_nosp, f)
train = [('MRLSTAQLIAIAYYMLSIGATVPQVDG', 'QGETEEALIQKRSYDYYQEPCDDYPQQQQQQEPCDYPQQQQQEEPCDYPQQQPQEPCDYPQQPQEPCDYPQQPQEPCDYPQQPQEPCDNPPQPDV', 121), ('MLTPRVLRALGWTGLFFLLLSPSNVLG', 'ASLSRDLETPPFLSFDPSNISINGAPLTEVPHAPSTESVSTNSESTNEHTITETTGKNAYIHNNASTDKQNANDTHKTPNILCDTEEVFVFLNET', 260)]
leng1 = 0
leng2 = 0
lst = []

for t in train:
    si, pr, sp = t
    leng1 = len(pr) - 10
    leng2 = len(pr) - 5
    lst.append(pr[:leng1])
    lst.append(pr[:leng2])
    lst.append(pr)

print(len(lst))
list(set(lst))
print(len(lst))<jupyter_output>6
6
{'fraction_non_alphanumeric': 0.11026548672566372, 'fraction_numerical': 0.034690265486725665, 'mean_word_length': 3.9396853146853146, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '472673', 'license': 'MIT', 'n_tokens_mistral': 2117, 'n_tokens_neox': 1897, 'n_words': 667, 'path': 'remote_generation/signal_peptide/code/1b_splits.ipynb', 'repo_name': 'fhalab/SPGen', 'size': '5650'}