<jupyter_start><jupyter_text>Data preprocessing  This notebook provides functions to format input data for sequence plottingThe formatted input data is returned as pandas dataframe containing following columns:* unique_protein_id: a single uniprot accession* modified_sequence: modified peptide sequence* naked_sequence: naked peptide sequence	* all_protein_ids: all UniProt IDs the peptide map to separated by ';'* start: peptide start position on protein sequence* end: peptide end position on protein sequence* PTMsites: list with PTM positions* PTMtypes: list with PTM types<jupyter_code>#hide

###### Generate input data for testing ######
#hide

import pandas as pd
import numpy as np
import re
#hide

test_df = pd.DataFrame(data={'all_protein_ids': ["A0A024R161;A0A087WT10;A0A087WTH1", 
                                                 "A0A024R161;A0A087WT10", 
                                                 "A0A087WTH5","A0A087WTH5",
                                                 "Nonsense"], 
                        'modified_sequence': ["PEPT[Phospho (STY)]IDER", 
                                              "SEQ[GlyGly (K)]UENCE[GlyGly (K)]R", 
                                              "VIEWER","NONSEQ",
                                              "NONSENSE"],
                        'naked_sequence': ["PEPTIDER", 
                                           "SEQUENCER", 
                                           "VIEWER","NONSEQ",
                                           "NONSENSE"]})
#hide

test_df_expanded = pd.DataFrame(data={'unique_protein_id': ["A0A024R161", "A0A087WT10", "A0A087WTH1", 
                                                            "A0A024R161", "A0A087WT10", 
                                                            "A0A087WTH5","A0A087WTH5",
                                                            "Nonsense"], 
                                      'modified_sequence': ["PEPT[Phospho (STY)]IDER", "PEPT[Phospho (STY)]IDER", "PEPT[Phospho (STY)]IDER",
                                                            "SEQ[GlyGly (K)]UENCE[GlyGly (K)]R", "SEQ[GlyGly (K)]UENCE[GlyGly (K)]R", 
                                                            "VIEWER","NONSEQ",
                                                            "NONSENSE"],
                                      'naked_sequence': ["PEPTIDER", "PEPTIDER", "PEPTIDER", 
                                                         "SEQUENCER", "SEQUENCER", 
                                                         "VIEWER","NONSEQ",
                                                         "NONSENSE"],
                                      'all_protein_ids': ["A0A024R161;A0A087WT10;A0A087WTH1", "A0A024R161;A0A087WT10;A0A087WTH1", "A0A024R161;A0A087WT10;A0A087WTH1", 
                                                 "A0A024R161;A0A087WT10", "A0A024R161;A0A087WT10", 
                                                 "A0A087WTH5","A0A087WTH5",
                                                 "Nonsense"]})
#hide

test_df_expanded_peptide_position = pd.DataFrame(data={'unique_protein_id': ["A0A024R161", "A0A087WT10", "A0A087WTH1", 
                                                                             "A0A024R161", "A0A087WT10", 
                                                                             "A0A087WTH5"], 
                                                       'modified_sequence': ["PEPT[Phospho (STY)]IDER", "PEPT[Phospho (STY)]IDER", "PEPT[Phospho (STY)]IDER",
                                                                             "SEQ[GlyGly (K)]UENCE[GlyGly (K)]R", "SEQ[GlyGly (K)]UENCE[GlyGly (K)]R", 
                                                                             "VIEWER"],
                                                       'naked_sequence': ["PEPTIDER", "PEPTIDER", "PEPTIDER", 
                                                                          "SEQUENCER", "SEQUENCER", 
                                                                          "VIEWER"],
                                                       'all_protein_ids': ["A0A024R161;A0A087WT10;A0A087WTH1", "A0A024R161;A0A087WT10;A0A087WTH1", "A0A024R161;A0A087WT10;A0A087WTH1", 
                                                                             "A0A024R161;A0A087WT10", "A0A024R161;A0A087WT10", 
                                                                             "A0A087WTH5"],
                                                       'start':[3,28,107,95,150,1],
                                                       'end':[10,35,114,103,158,6]})
#hide

test_df_modifications = pd.DataFrame(data={'unique_protein_id': ["A0A024R161", "A0A087WT10", "A0A087WTH1", 
                                                                             "A0A024R161", "A0A087WT10", 
                                                                             "A0A087WTH5"], 
                                                       'modified_sequence': ["PEPT[Phospho (STY)]IDER", "PEPT[Phospho (STY)]IDER", "PEPT[Phospho (STY)]IDER",
                                                                             "SEQ[GlyGly (K)]UENCE[GlyGly (K)]R", "SEQ[GlyGly (K)]UENCE[GlyGly (K)]R", 
                                                                             "VIEWER"],
                                                       'naked_sequence': ["PEPTIDER", "PEPTIDER", "PEPTIDER", 
                                                                          "SEQUENCER", "SEQUENCER", 
                                                                          "VIEWER"],
                                           'all_protein_ids': ["A0A024R161;A0A087WT10;A0A087WTH1", "A0A024R161;A0A087WT10;A0A087WTH1", "A0A024R161;A0A087WT10;A0A087WTH1", 
                                                                             "A0A024R161;A0A087WT10", "A0A024R161;A0A087WT10", 
                                                                             "A0A087WTH5"],
                                                       'start':[3,28,107,95,150,1],
                                                       'end':[10,35,114,103,158,6], 
                                           'PTMsites':[[3],[3],[3],[2,7],[2,7],[]],
                                           'PTMtypes':[["[Phospho (STY)]"],["[Phospho (STY)]"],["[Phospho (STY)]"],["[GlyGly (K)]","[GlyGly (K)]"],["[GlyGly (K)]","[GlyGly (K)]"],[]]})<jupyter_output><empty_output><jupyter_text>Split protein group into unique protein accessions The *all_protein_ids* column in *df* is split by ';' to result in unique rows for each UniProt accession. The exploded dataframe has a new column *unique_protein_id*.<jupyter_code>#export
import pandas as pd
def expand_protein_ids(df: pd.DataFrame):
    """
    Function to split protein groups in 'all_protein_ids' by ';' into separate rows.
    The resulting dataframe has a new column 'unique_protein_id'.
    
    Args:
        df (pd.DataFrame): Experimental data that was imported by the 'import_data' function.
    Returns:
        pd.DataFrame: Exploded dataframe with a new column 'unique_protein_id'.
        
    """
    df = df.copy(deep=True)
    df.all_protein_ids = df.all_protein_ids.str.split(';')
    df["all_protein_ids_all"] = df.all_protein_ids.apply(lambda x: ';'.join(sorted(x)))
    res = df.explode('all_protein_ids').reset_index(drop=True)
    res.columns = ['unique_protein_id','modified_sequence','naked_sequence','all_protein_ids']
    return res
#hide

def test_expand_protein_ids():
    res = expand_protein_ids(test_df)
    pd.testing.assert_frame_equal(res,test_df_expanded)
    
test_expand_protein_ids()<jupyter_output><empty_output><jupyter_text>Annotate peptides with start and end position The *get_peptide_position* function annotates a peptide's start and end position in the given protein sequence.<jupyter_code>#export 
import re
import numpy as np
from pyteomics import fasta

def pep_position_helper(seq: str, prot: str, fasta: fasta, verbose: bool = True):
    """
    Helper function for 'get_peptide_position'.
    
    Args:
        seq (str): Naked peptide sequence.
        prot (str): UniProt protein accession.
        fasta (fasta): Fasta file imported by pyteomics 'fasta.IndexedUniProt'.
        verbose (bool, optional): Flag to print warnings if no matching sequence is found for a protein in the provided fasta. Defaults to 'True'.
    Returns:
        [int, int]: int: peptide start position, int: peptide end position
        
    """
    try:
        fasta_prot = fasta[prot]
        search_res = re.search(seq,fasta_prot.sequence)
        if search_res is None:
            start, end = np.NaN, np.NaN
            if verbose:
                warnings.warn(f'Peptide sequence {seq} could not be mached to {prot} in the selected fasta.')
        else:
            start, end = search_res.span()
    except:
        start, end = np.NaN, np.NaN 
        if verbose:
            warnings.warn(f'No matching entry for {prot} in the selected fasta.')

    return start, end-1

#hide

# @ToDO: Maybe write .fasta import function to remove dependency for pyteomics
test_fasta = fasta.IndexedUniProt("../testdata/test.fasta")

def test_pep_position_helper():
    start, end = pep_position_helper("PEPTIDER","A0A024R161",test_fasta)
    np.testing.assert_equal([start, end], [3,10])

test_pep_position_helper()
#export 

import warnings

def get_peptide_position(df: pd.DataFrame, fasta: fasta, verbose:bool = True):
    """
    Function to get start and end position of each peptide in the given protein.
    
    Args:
        df (pd.DataFrame): Experimental data that was imported by the 'import_data' function and processed by 'expand_protein_ids'.
        fasta (fasta): Fasta file imported by pyteomics 'fasta.IndexedUniProt'.
        verbose (bool, optional): Flag to print warnings if no matching sequence is found for a protein in the provided fasta. Defaults to 'True'.
    Returns:
        pd.DataFrame: Dataframe with a new columns 'start' and 'end', indicating the start and end index position of the peptide sequence.
        
    """
    res = df.copy(deep=True)
    res[['start','end']] = res.apply(lambda row: pep_position_helper(row['naked_sequence'], 
                                                                     row['unique_protein_id'], 
                                                                     fasta,
                                                                     verbose=verbose), 
                                     axis=1, result_type='expand')
    
    res_na = res[res.isnull().any(1)]
    prots_na = res_na.unique_protein_id.unique()
    
    res = res.dropna()
    res['start'] = res['start'].astype('int64')
    res['end'] = res['end'].astype('int64')
    return res
#hide

def test_get_peptide_position():
    with warnings.catch_warnings(record=True) as w:
        res = get_peptide_position(test_df_expanded, test_fasta)
        assert len(w) == 2
        assert "Peptide sequence NONSEQ could not be mached" in str(w[0].message)
        assert "No matching entry for Nonsense" in str(w[1].message)
    pd.testing.assert_frame_equal(res,test_df_expanded_peptide_position)
    
test_get_peptide_position()<jupyter_output><empty_output><jupyter_text>Annotate each peptide with PTM site indices and modification types The *get_modifications* function annotates sequence positions and modification types of all PTMs on a peptide.<jupyter_code>#export
import numpy as np

def get_ptm_sites(peptide: str, modification_reg: str):
    """
    Function to get sequence positions of all PTMs of a peptide in the given protein.
    
    Args:
        peptide (str): Experimental data that was imported by the 'import_data' function and processed by 'expand_protein_ids'.
        modification_reg (str): Regular expression for the modifications. 
    Returns:
        list: List of integers with PTM site location indices on the peptide.
        
    """
    r = re.compile(modification_reg)
    starts=[]
    ends=[]
    for m in r.finditer(peptide):
        starts.append(m.start())
        ends.append(m.end())
    PTM_sites = np.zeros(len(starts))
    for idx in range(0,len(starts)):
        if idx > 0:
            previous_len=previous_len+(ends[idx-1]-starts[idx-1])
        else:
            previous_len=0
        PTM_sites[idx] = starts[idx] - previous_len - 1
    return [int(i) for i in PTM_sites]


#hide

def test_get_ptm_sites():
    myPep = "PEPT[Phospho]IDE[GlyGly (K)]R"
    res = get_ptm_sites(myPep, modification_reg=r'\[.*?\]')
    np.testing.assert_equal(res, [3,6])
    
test_get_ptm_sites()
#export
import re

def get_modifications(df: pd.DataFrame, mod_reg: str):
    """
    Function to get sequence positions and modification types of all PTMs of a peptide in the given protein.
    
    Args:
        df (pd.DataFrame): Experimental data that was imported by the 'import_data' function and processed by 'expand_protein_ids'.
        mod_reg (str): Regular expression for the modifications. 
    Returns:
        pd.DataFrame: Dataframe with a new columns 'PTMsites' and 'PTMtypes' containing lists of PTM site indices and modification types, respectively.
        
    """
    res = df.copy(deep=True)
    res['PTMsites'] = res.apply(lambda row: get_ptm_sites(row['modified_sequence'], 
                                                        modification_reg=mod_reg), axis=1)
    res['PTMtypes'] = res.apply(lambda row: re.findall(mod_reg, row['modified_sequence']), axis=1)
    return res
#hide

def test_get_modifications():
    res = get_modifications(test_df_expanded_peptide_position, mod_reg = r'\[.*?\]')
    pd.testing.assert_frame_equal(res, test_df_modifications)
    
test_get_modifications()<jupyter_output><empty_output><jupyter_text>Preprocessing wrapper The *format_input_data* function is a wrapper for all previous functions described in this notebook. It calls following functios:- *expand_protein_ids*: to split protein groups into unique uniprot accessions- *get_peptide_position*: to annotate each peptide with its start and end position on its respective protein sequence- *get_modifications*: To get the sequence positions and modification types of all PTMs of a peptide.<jupyter_code>#export

def format_input_data(df: pd.DataFrame, fasta: fasta, modification_exp: str, verbose:bool = True):
    """
    Function to format input data and to annotate sequence start and end positions plus PTM sites.
    
    Args:
        df (pd.DataFrame): Experimental data that was imported by the 'import_data' function.
        fasta (fasta): Fasta file imported by pyteomics 'fasta.IndexedUniProt'.
        modification_exp (str): Regular expression for the modifications. 
        verbose (bool, optional): Flag to print warnings if no matching sequence is found for a protein in the provided fasta. Defaults to 'True'.
    Returns:
        pd.DataFrame: Dataframe with unique uniprot accessions, sequence start and end positions, and PTM site information.
    
    """
    res = df.copy(deep=True)
    res = expand_protein_ids(res)
    res = get_peptide_position(res, fasta = fasta, verbose=verbose)
    res = get_modifications(res, mod_reg = modification_exp)
    return res
#hide

def test_format_input_data():
    with warnings.catch_warnings(record=True) as w:
        res = format_input_data(df=test_df, fasta = test_fasta, modification_exp = r'\[.*?\]')
        assert len(w) == 2
        assert "Peptide sequence NONSEQ could not be mached" in str(w[0].message)
        assert "No matching entry for Nonsense" in str(w[1].message)   
    pd.testing.assert_frame_equal(res, test_df_modifications)

test_format_input_data()
#hide

###### Export notebook to script ###### 
#hide
from nbdev.showdoc import *
#hide
from nbdev.export import *
notebook2script()<jupyter_output>Converted Importing.ipynb.
Converted index.ipynb.
Converted organisms_data.ipynb.
Converted Preprocessing.ipynb.
Converted proteolytic_cleavage.ipynb.
Converted SequencePlot.ipynb.
Converted Uniprot_integration.ipynb.
{'fraction_non_alphanumeric': 0.09803191159123363, 'fraction_numerical': 0.031476997578692496, 'mean_word_length': 1.8540042523033309, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 129, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '51189', 'license': 'Apache-2.0', 'n_tokens_mistral': 4905, 'n_tokens_neox': 4386, 'n_words': 1106, 'path': 'nbs/Preprocessing.ipynb', 'repo_name': 'JuliaS92/alphamap-1', 'size': '16107'}