<jupyter_start><jupyter_text>This has been implemented in mixed order and should only be used as a guide. Preliminary preprocessing has also been done at the end<jupyter_code>import pandas as pd

df = pd.read_csv("C:/Users/nik00/py/proj/humanPPI-ids-uniprot.txt", sep="\t", header=None)
df[1].value_counts()<jupyter_output><empty_output><jupyter_text>We start by getting all protein identifiers in test and train data<jupyter_code>x = set(pd.unique(df[0]))
x = x| set(pd.unique(df[1]))
len(x)
df = pd.read_csv("C:/Users/nik00/py/proj/hyppi-independent.txt", sep="\t")
df.head()
x = x| set(pd.unique(df.ID1))
len(x)
x = x| set(pd.unique(df.ID2))
len(x)

df = pd.read_csv("C:/Users/nik00/py/proj/hyppi-train.txt", sep="\t")
x = x| set(pd.unique(df.ID1))
len(x)
x = x| set(pd.unique(df.ID2))
len(x)
x = list(x)
x[1905]
len(set(x))
count = 0
for i in x:
    if len(i) >6:
        count+=1
        print(i)
        if count>20:
            break
count
from Bio import SeqIO
l=[]
for seq_record in SeqIO.parse("C:/Users/nik00/py/proj/uniprot_sprot.fasta", "fasta"):
    l.append(seq_record.id.split('|')[1])
count = 0
for i in ll:
    if i not in l:
        count+=1
count
from Bio import SeqIO
l=[]
for seq_record in SeqIO.parse("C:/Users/nik00/py/proj/uniprot_sprot.fasta", "fasta"):
#     print(seq_record.id)
#     print(repr(seq_record.seq))
#     print(len(seq_record))
    l.append(seq_record.id.split('|')[1])
len(l)
'O00487' in l
import requests as r
from Bio import SeqIO
from io import StringIO

cID='A0A151PMG2'

baseUrl="http://www.uniprot.org/uniprot/"
currentUrl=baseUrl+cID+".fasta"
response = r.post(currentUrl)
cData=''.join(response.text)

Seq=StringIO(cData)
pSeq=list(SeqIO.parse(Seq,'fasta'))

s = pSeq[0].seq
pSeq[0].description
s
#if no response,  len(pSeq) == 0
pSeq[0]
#gene pSeq[0].id.split('|')[2]
gene = pSeq[0].id.split('|')[2] #dont use gene
line = pSeq[0].description
line = line[line.find(gene)+len(gene)+1:]
Protein = line[:line.find('OS=')-1]
line = line[len(Protein)+1:]
OrganismName = line[3:line.find('OX=')-1]
OrganismIdentifier = line[3:line.find('OX=')-1]
line = line[line.find('OX='):]
OrganismIdentifier = line[3:line.find('GN=')-1]
line = line[line.find('GN='):]
GeneName = line[3:line.find('PE=')-1]
line = line[line.find('PE='):]
ProteinExistence = line[3:line.find('SV=')-1]
line = line[line.find('SV='):]
SequenceVersion = line[3:]
print('SequenceVersion',SequenceVersion)
print('ProteinExistence',ProteinExistence)
print('GeneName',GeneName)
print('OrganismName',OrganismName)
print('OrganismIdentifier',OrganismIdentifier)
print('Protein',Protein)<jupyter_output>SequenceVersion 1
ProteinExistence 4
GeneName yiaD
OrganismName Yersinia pestis
OrganismIdentifier 632
Protein OmpA family lipoprotein<jupyter_text>Create DATA file that has several attributes for each protein we get by web scraping<jupyter_code>COLUMN_NAMES = ['id','Sequence','SequenceVersion','Protein','ProteinExistence','GeneName','OrganismName','OrganismIdentifier']
df = pd.DataFrame(columns=COLUMN_NAMES)
nf = []
df.to_pickle("C:/Users/nik00/py/proj/DATA.pkl")
df = pd.read_pickle("C:/Users/nik00/py/proj/DATA.pkl")
df.head()


import requests as r
from Bio import SeqIO
from io import StringIO

#extract data for each protein identifier
baseUrl="http://www.uniprot.org/uniprot/"
df = pd.read_pickle("C:/Users/nik00/py/proj/DATA.pkl")
count = -1

for i in x[df.shape[0]:]:
    count+=1
    currentUrl=baseUrl+i+".fasta"
    response = r.post(currentUrl)
    cData=''.join(response.text)
    Seq=StringIO(cData)
    pSeq=list(SeqIO.parse(Seq,'fasta'))
    if len(pSeq) == 0:
        nf.append(i)
        df = df.append({'id': i}, ignore_index=True)
        continue
    
    gene = pSeq[0].id.split('|')[2] #dont use gene
    line = pSeq[0].description
    line = line[line.find(gene)+len(gene)+1:]
    Protein = line[:line.find('OS=')-1]
    line = line[len(Protein)+1:]
    OrganismName = line[3:line.find('OX=')-1]
    OrganismIdentifier = line[3:line.find('OX=')-1]
    line = line[line.find('OX='):]
    OrganismIdentifier = line[3:line.find('GN=')-1]
    line = line[line.find('GN='):]
    GeneName = line[3:line.find('PE=')-1]
    line = line[line.find('PE='):]
    ProteinExistence = line[3:line.find('SV=')-1]
    line = line[line.find('SV='):]
    SequenceVersion = line[3:]    
    df = df.append({'id': i,'Sequence': pSeq[0].seq,'SequenceVersion':SequenceVersion, 'Protein':Protein, 'ProteinExistence':ProteinExistence, 'GeneName':GeneName, 'OrganismName':OrganismName, 'OrganismIdentifier':OrganismIdentifier}, ignore_index=True)
    
    if count%10 == 0:
        df.to_pickle("C:/Users/nik00/py/proj/DATA.pkl")
    
    
    
len(nf)
#df.head(150)
'Q8NGG4'
df = pd.read_pickle("C:/Users/nik00/py/proj/DATA.pkl")
df.shape[0]
df.iloc[50:60,:]
df.head(150)

#df = df.set_index('id')
'A0A151PMG2' in list(df['id'])
df.shape[0]
outdated = []
#df = df.set_index('id')
outdated = df[df.isna().any(axis=1)].index
outdated
URL = 'https://www.uniprot.org/uniparc/?query='
i = 'A0A151PQD1'
page = r.get(URL+i)
content = str(page.content)
content = content[content.find('entryID')+7:]
link = content[content.find('<a href="')+9:content.find('">UPI')]
new_URL = 'https://www.uniprot.org'+link+'.fasta'
OS = content[content.find('<td>')+4:content.find('<br/>')]
print(OS)
new_URL
df.loc['A0A151PQD1']['OrganismName']
URL = 'https://www.uniprot.org/uniparc/?query='
#Get sequences of the outdated protein identifiers
for i in outdated:
    page = r.get(URL+i)
    content = str(page.content)
    content = content[content.find('entryID')+7:]
    link = content[content.find('<a href="')+9:content.find('">UPI')]
    new_URL = 'https://www.uniprot.org'+link+'.fasta'
    OS = content[content.find('<td>')+4:content.find('<br/>')]
    response = r.post(new_URL)
    cData=''.join(response.text)
    Seq=StringIO(cData)
    pSeq=list(SeqIO.parse(Seq,'fasta'))
    #print(pSeq[0].seq)
    if len(pSeq) == 0:
        nf.append(i)
        continue
    df.loc[i]['Sequence'] = pSeq[0].seq
    df.loc[i]['OrganismName'] = OS
df = pd.read_pickle("C:/Users/nik00/py/proj/DATA.pkl")
'fsaffag' not in df.index
pd.unique(df["OrganismName"])
#df.head()
count = 0
for i in df.index:
    count+=1
    if df.loc[i]["OrganismName"].find("Yersinia") != -1:
        df.loc[i]["OrganismName"] = 'Yersinia pestis'
    if df.loc[i]["OrganismName"].find("Homo sapiens") != -1:
        df.loc[i]["OrganismName"] = 'Homo sapiens'
    
count
pd.unique(df["OrganismName"])
df.to_pickle("C:/Users/nik00/py/proj/DATA.pkl")
import pandas as pd
data = pd.read_pickle("C:/Users/nik00/py/proj/DATA.pkl")
data.describe()
len(list(data.loc["A6NFA1"]["Sequence"]))<jupyter_output><empty_output><jupyter_text>Get amino acid sequences from train dataset and store it as a dataframe<jupyter_code>df = pd.read_csv("C:/Users/nik00/py/proj/hyppi-train.txt", sep="\t")
idd = df.loc[0]["ID1"]
df["ID1"]
organisms = []
for idd in df["ID2"]:
    if idd in data.index:
        organisms.append(data.loc[idd]["OrganismName"])
    else:
        print("Not found")
set(organisms)
not_Yersinia=[]
for idd in df["ID2"]:
    org = data.loc[idd]["OrganismName"]
    if org != 'Yersinia pestis':
        not_Yersinia.append(idd)
not_Yersinia
l1=list("anad")
l2 = list("bnuqe")
' '.join(l1+l2)
df.iloc[0]["ID1"]
data.loc[df.iloc[row]["ID1"]]["Sequence"]
df["Human"] = [list(data.loc[df.iloc[row]["ID1"]]["Sequence"]) for row in range(df.shape[0])]
df["Yersinia"] = [list(data.loc[df.iloc[row]["ID2"]]["Sequence"]) for row in range(df.shape[0])]
df["Joined"] = [df.loc[row]['Human']+df.loc[row]['Yersinia'] for row in range(df.shape[0])]
df.head()
#print(data.loc['B0I1T2']["Sequence"])
' '.join(df.loc[0]['Human'])

df.to_pickle("C:/Users/nik00/py/proj/hyppi-train.pkl")
import numpy as np
try:
    with open("/content/drive/MyDrive/ML_Data/hyppi-train.pkl", "rb") as fh:
      df = pickle.load(fh)
except:
    df = pd.read_pickle("C:/Users/nik00/py/proj/hyppi-train.pkl")
df = df.sample(frac=1).reset_index(drop=True)[['Human','Yersinia','Joined','label']]   #SHUFFLE
df['label'] = np.where(df['label']==-1, 0, 1)
df.head()
df.to_pickle("C:/Users/nik00/py/proj/hyppi-train.pkl")<jupyter_output><empty_output><jupyter_text>Get amino acid sequences from test dataset and store it as a dataframe<jupyter_code>df = pd.read_csv("C:/Users/nik00/py/proj/hyppi-independent.txt", sep="\t")
idd = df.loc[0]["ID1"]
df["ID1"]
df
organisms = []
for idd in df["ID1"]:
    if idd in data.index:
        organisms.append(data.loc[idd]["OrganismName"])
    else:
        print("Not found")
set(organisms)
df["Human"] = [list(data.loc[df.iloc[row]["ID1"]]["Sequence"]) for row in range(df.shape[0])]
df["Yersinia"] = [list(data.loc[df.iloc[row]["ID2"]]["Sequence"]) for row in range(df.shape[0])]
df["Joined"] = [df.loc[row]['Human']+df.loc[row]['Yersinia'] for row in range(df.shape[0])]
df.to_pickle("C:/Users/nik00/py/proj/hyppi-independent.pkl")
try:
    with open("/content/drive/MyDrive/ML_Data/hyppi-train.pkl", "rb") as fh:
      df = pickle.load(fh)
except:
    df = pd.read_pickle("C:/Users/nik00/py/proj/hyppi-independent.pkl")
df = df.sample(frac=1).reset_index(drop=True)[['Human','Yersinia','Joined','label']]   #SHUFFLE
df['label'] = np.where(df['label']==-1, 0, 1)
df
df.to_pickle("C:/Users/nik00/py/proj/hyppi-independent.pkl")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2029203349796006, 'fraction_numerical': 0.022331973373416362, 'mean_word_length': 4.929344366645449, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 28, 'https://': 4, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '49141', 'license': 'MIT', 'n_tokens_mistral': 3960, 'n_tokens_neox': 3751, 'n_words': 616, 'path': 'Web_scraping.ipynb', 'repo_name': 'nikhil-mathews/MastersPr_Predicting-Human-Pathogen-PPIs-using-Natural-Language-Processing-methods', 'size': '9314'}