<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import os
home_dir = '../'
os.chdir(home_dir)
from ext import download_pubmed_articles, search_gene_id, download_gene_names
from norm import normalize_genes
from relation_finder import find_sentences

"""
Required: A valid entrez registered email and entrez api-key.
More information: Entrez Programming Utilities (E-utilities)
"""  
from config import entrez_api_key, entrez_email
import pandas as pd
import numpy as np
import re
from Bio import Entrez
import xmltodict<jupyter_output><empty_output><jupyter_text>Preprocessing BioGRID.  Biological General Repository for Interaction Datasets<jupyter_code>biogrid = pd.read_csv('Data/BIOGRID-ALL-4.3.196.mitab.txt', sep = '\t')
biogrid = biogrid[(biogrid['#ID Interactor A'].str[:6] == 'entrez') & (biogrid['ID Interactor B'].str[:6] == 'entrez')]
biogrid['source'] = biogrid['Alt IDs Interactor A'].apply(lambda x: re.findall(r'entrez gene/locuslink:[\w]*', x)[0][22:])
biogrid['target'] = biogrid['Alt IDs Interactor B'].apply(lambda x: re.findall(r'entrez gene/locuslink:[\w]*', x)[0][22:])
biogrid['interaction'] = biogrid['Interaction Types'].str[17:-1]
biogrid['pubmed'] = biogrid['Publication Identifiers'].str[7:]
biogrid = biogrid[~biogrid.pubmed.isna()]
biogrid.head(5)
print('No of entries in BioGRID:', biogrid.shape[0])
# Create a list of all Pubmed IDs
pubmed_list = list(set(biogrid['Publication Identifiers'].str[7:].to_list()))
# Download abstracts from Pubmed IDs
print('Set 1')
res1 = download_pubmed_articles(entrez_api_key, entrez_email, pubmed_list[:10000])
print('Set 2')
res2 = download_pubmed_articles(entrez_api_key, entrez_email, pubmed_list[10000:20000])
print('Set 3')
res3 = download_pubmed_articles(entrez_api_key, entrez_email, pubmed_list[20000:30000])
print('Set 4')
res4 = download_pubmed_articles(entrez_api_key, entrez_email, pubmed_list[30000:40000])
print('Set 5')
res5 = download_pubmed_articles(entrez_api_key, entrez_email, pubmed_list[40000:50000])
print('Set 6')
res6 = download_pubmed_articles(entrez_api_key, entrez_email, pubmed_list[50000:60000])
print('Set 7')
res7 = download_pubmed_articles(entrez_api_key, entrez_email, pubmed_list[60000:])
res = res1 + res2 + res3 + res4 + res5 + res6 + res7
print('-'*15)
print(f'{len(res)} articles found out of {len(pubmed_list)}')
# Create DataFrame for downloaded pubmed articles.
pubmed_articles = pd.DataFrame(res).dropna()
pubmed_articles.index = pubmed_articles.pubmed
pubmed_articles.index.name = 'pmid'
pubmed_articles.head(5)
# Merge BioGRID with pubmed dataframe
biogrid.index = biogrid.pubmed
biogrid.index.name = 'pmid'
biogrid = biogrid[['source', 'target', 'interaction']]
df = biogrid.join(pubmed_articles).dropna(subset = ['abstract'])
df = df[['source', 'target', 'interaction', 'abstract']]
df.head(5)
# Search gene names
gene_name_list = list(set(df.source.to_list() + df.target.to_list()))
gene_name_list = [i for i in gene_name_list if len(i)>0]
gene_search_res = search_gene_id(entrez_api_key, entrez_email, gene_name_list)
# Download genes.
gene_ids = list(set([i['id'] for i in gene_search_res]))
print('Set 1')
gene_dict1 = download_gene_names(entrez_api_key, entrez_email, gene_ids[:10000])
print('Set 2')
gene_dict2 = download_gene_names(entrez_api_key, entrez_email, gene_ids[10000:20000])
print('Set 3')
gene_dict3 = download_gene_names(entrez_api_key, entrez_email, gene_ids[20000:30000])
print('Set 4')
gene_dict4 = download_gene_names(entrez_api_key, entrez_email, gene_ids[30000:40000])
print('Set 5')
gene_dict5 = download_gene_names(entrez_api_key, entrez_email, gene_ids[40000:])
gene_dict = gene_dict1 + gene_dict2 + gene_dict3 + gene_dict4 + gene_dict5
print('-'*15)
print(f'{len(gene_dict)} genes downloaded out of {len(gene_ids)}')
# Normalize genes
df['abstract'] = normalize_genes(gene_dict, df.source.to_list(), df.target.to_list(), df.abstract.to_list())
df.head(5)
# Find sentences with gene mentions
rel_df = find_sentences(df)
rel_df.head(5)
rel_df = rel_df[['source', 'target', 'interaction', 'sentence', 'match']]
rel_df.to_csv('biogrid_norm_rel.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14095422620489223, 'fraction_numerical': 0.03996124969726326, 'mean_word_length': 8.322799097065463, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '435764', 'license': 'Unlicense', 'n_tokens_mistral': 1663, 'n_tokens_neox': 1529, 'n_words': 306, 'path': 'Preprocessing/BioGRID.ipynb', 'repo_name': 'srivastavaprashant/ARM', 'size': '4129'}