<jupyter_start><jupyter_text>Downloads PDB Structure Information**[Work in progress]**This notebook downloads 3D-structure information from the Worldwide Protein Data Bank.Data sources: [Protein Data Bank Japan](https://pdbj.org/), [PDBe Protein Data Bank Europe](https://www.ebi.ac.uk/pdbe/)Author: <NAME> (<EMAIL>)<jupyter_code>import os
import re
import numpy as np
import pandas as pd
import dateutil
import urllib
from pathlib import Path
pd.options.display.max_rows = None  # display all rows
pd.options.display.max_columns = None  # display all columsns
NEO4J_IMPORT = Path(os.getenv('NEO4J_IMPORT'))
print(NEO4J_IMPORT)<jupyter_output>/Users/peter/Library/Application Support/com.Neo4j.Relate/data/dbmss/dbms-8bf637fc-0d20-4d9f-9c6f-f7e72e92a4da/import<jupyter_text>Get PDB summary data<jupyter_code>URL = 'https://pdbj.org/rest/newweb/search/sql?format=csv&q='
sql_query = 'SELECT pdbid, release_date AS releasedate, pdbx_descriptor AS description, struct_title AS title, exptl_method AS methods, resolution FROM brief_summary'
encoded_query = urllib.parse.quote(sql_query)
summary_url = URL + encoded_query
df_summary = pd.read_csv(summary_url, dtype=str)
df_summary.head()
df_summary.rename(columns={'pdbid': 'pdbId'}, inplace=True)
df_summary['pdbId'] = df_summary['pdbId'].str.upper()
df_summary.rename(columns={'releasedate': 'releaseDate'}, inplace=True)
print('Number of records:', df_summary.shape[0])
df_summary.head()<jupyter_output>Number of records: 178451<jupyter_text>Get refinement data<jupyter_code>sqlQuery = 'SELECT "ls_R_factor_R_free", entry_id FROM refine'
encoded_query = urllib.parse.quote(sqlQuery)
refine_url = URL + encoded_query
#df_refine = pd.read_csv(refine_url, usecols=['ls_R_factor_R_free', 'entry_id'], dtype=str)
df_refine = pd.read_csv(refine_url, dtype=str)
df_refine.head()
df_refine.rename(columns={'ls_R_factor_R_free': 'rFree', 'entry_id': 'pdbId'}, inplace=True)
df_refine.fillna('', inplace=True)
print('Number of records:', df_refine.shape[0])
df_refine.sample(5)
structures = df_summary.merge(df_refine, on='pdbId', how='left')
structures.fillna('', inplace=True)
structures.drop_duplicates(subset=['pdbId'], inplace=True)
print('Number of structures:', structures.shape[0])
structures.sample(10)
structures['pdbId'] = 'pdb:' + structures['pdbId']

# convert methods string to a semicolon separated list (our default one-to-many representation in CSV files)
structures['methods'] = structures['methods'].str.replace('[', '', regex=False)
structures['methods'] = structures['methods'].str.replace(']', '', regex=False)
structures['methods'] = structures['methods'].str.replace("'", '', regex=False)
structures['methods'] = structures['methods'].str.replace(',', ';', regex=False)
structures['methods'] = structures['methods'].str.replace('; ', ';', regex=False)
structures['methods'].unique()
structures.head()
structures.to_csv(NEO4J_IMPORT / "01f-PDBStructure.csv", index=False)<jupyter_output><empty_output><jupyter_text>Get PDB Entity info<jupyter_code>sqlQuery = "select pdbid, id, pdbx_description from entity"
encodedSQL = urllib.parse.quote(sqlQuery)
entity_url = URL + encodedSQL
df_entity = pd.read_csv(entity_url, dtype=str)
df_entity.rename(columns={'pdbid': 'pdbId', 'id': 'entityId', 'pdbx_description': 'description'}, inplace=True)
df_entity['pdbId'] = df_entity['pdbId'].str.upper()
print('Number of entities:', df_entity.shape[0])
df_entity.head()
def categorize(description):
    words = re.split('[ ,\-()]', str(description).lower())
    
    keywords_ab = ['antibody', 'fab', 'nanobody']
    keywords_hc = ['heavy', 'h', 'hc']
    keywords_lc = ['light', 'l', 'lc']
    
    category = ''
    
    if any(w in words for w in keywords_ab):
        category = 'antibody'
        if any(w in words for w in keywords_hc):
            category = 'antibody heavy chain'
            
        if any(w in words for w in keywords_lc):
            category = 'antibody light chain'
    
    return category
df_entity['category'] = df_entity['description'].apply(categorize)
df_entity.query("category != ''").head()<jupyter_output><empty_output><jupyter_text>Get PDB Polymer Entities<jupyter_code>sqlQuery = "select pdbid, entity_id, pdbx_strand_id, type, pdbx_seq_one_letter_code_can from entity_poly"
encodedSQL = urllib.parse.quote(sqlQuery)
poly_url = URL + encodedSQL
df_poly = pd.read_csv(poly_url, dtype=str)
df_poly.rename(columns={'pdbid': 'pdbId', 'entity_id': 'entityId', 'pdbx_seq_one_letter_code_can': 'sequence'}, inplace=True)
df_poly['pdbId'] = df_poly['pdbId'].str.upper()
df_poly['chainId'] = df_poly['pdbx_strand_id'].str.split(',')
df_poly = df_poly.explode('chainId')
df_poly = df_poly[['pdbId', 'entityId', 'chainId', 'type', 'sequence']]
print("Number of polymer entities:", df_poly.shape[0])
df_poly.head()
df_chain = pd.merge(df_entity, df_poly, on=['pdbId', 'entityId'])
print('Number of polymer chains:', df_chain.shape[0])
df_chain.head()<jupyter_output><empty_output><jupyter_text>Get PDB Chain - UniProt sequence mappings<jupyter_code>sifts_url = 'http://ftp.ebi.ac.uk/pub/databases/msd/sifts/flatfiles/tsv/uniprot_segments_observed.tsv.gz'
chains = pd.read_csv(sifts_url, sep='\t', skiprows=1, dtype=str)
print("Number of chains:", chains.shape[0])
chains.head()
chains.rename(columns={'PDB': 'pdbId', 'CHAIN': 'chainId', 'SP_PRIMARY': 'accession'}, inplace=True)
chains.rename(columns={'RES_BEG': 'seqresStart', 'RES_END': 'seqresEnd'}, inplace=True)
chains.rename(columns={'PDB_BEG': 'pdbStart', 'PDB_END': 'pdbEnd'}, inplace=True)
chains.rename(columns={'SP_BEG': 'uniprotStart', 'SP_END': 'uniprotEnd'}, inplace=True)
chains['pdbId'] = chains['pdbId'].str.upper()
chains.head()<jupyter_output><empty_output><jupyter_text>Sort chains by uniprot residue number<jupyter_code>chains['uniprotStart'] = chains['uniprotStart'].astype(int)
chains['uniprotEnd'] = chains['uniprotEnd'].astype(int)
chains['length'] = chains['uniprotEnd'] - chains['uniprotStart'] + 1
# TODO length seems to be string??
chains['length'] = chains['length'].astype(int)
#
chains.sort_values(by='uniprotStart', inplace=True)
chains['uniprotStart'] = chains['uniprotStart'].astype(str)
chains['uniprotEnd'] = chains['uniprotEnd'].astype(str)
chains.head()
chains.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 839171 entries, 0 to 345771
Data columns (total 10 columns):
 #   Column        Non-Null Count   Dtype 
---  ------        --------------   ----- 
 0   pdbId         839171 non-null  object
 1   chainId       839129 non-null  object
 2   accession     839171 non-null  object
 3   seqresStart   839171 non-null  object
 4   seqresEnd     839171 non-null  object
 5   pdbStart      839171 non-null  object
 6   pdbEnd        839171 non-null  object
 7   uniprotStart  839171 non-null  object
 8   uniprotEnd    839171 non-null  object
 9   length        839171 non-null  int64 
dtypes: int64(1), object(9)
memory usage: 70.4+ MB<jupyter_text>Group data by PDB chains<jupyter_code>chains = chains.groupby(['pdbId','chainId','accession']).agg(list).reset_index()<jupyter_output><empty_output><jupyter_text>Create semicolon separated string of residue numbers so they can be represented in a csv file<jupyter_code>chains['uniprotStart'] = chains['uniprotStart'].apply(lambda x: ';'.join(x))
chains['uniprotEnd'] = chains['uniprotEnd'].apply(lambda x: ';'.join(x))
chains['seqresStart'] = chains['seqresStart'].apply(lambda x: ';'.join(x))
chains['seqresEnd'] = chains['seqresEnd'].apply(lambda x: ';'.join(x))
chains['pdbStart'] = chains['pdbStart'].apply(lambda x: ';'.join(x))
chains['pdbEnd'] = chains['pdbEnd'].apply(lambda x: ';'.join(x))
chains['residues'] = chains['length'].apply(lambda x: sum(x))
print("Number of chains with UniProt mapping:", chains.shape[0])
chains.tail()
chains.query('pdbId == "6XDG"')
chains = df_chain.merge(chains, on=['pdbId', 'chainId'], how='left')
chains.fillna('', inplace=True)<jupyter_output><empty_output><jupyter_text>Assign CURIES<jupyter_code>chains['accession'] = chains['accession'].apply(lambda x: x if x == '' else 'uniprot:' + x)
chains['pdbId'] = 'pdb:' + chains['pdbId']
chains['pdbChainId'] = chains['pdbId'] + '.' + chains['chainId']
print("Total number of polymer chains:", chains.shape[0])
chains.sample(10)
chains.to_csv(NEO4J_IMPORT / "01f-PDBChain.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.164272030651341, 'fraction_numerical': 0.017600574712643677, 'mean_word_length': 6.498204667863555, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '464616', 'license': 'MIT', 'n_tokens_mistral': 3112, 'n_tokens_neox': 2973, 'n_words': 567, 'path': 'notebooks/dataprep/01f-PDBStructure.ipynb', 'repo_name': 'yogeshchaudhari/covid-19-community', 'size': '8364'}