<jupyter_start><jupyter_text>GFFcompare analyses Import required modules<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
from matplotlib_venn import venn3
from upsetplot import from_contents, UpSet, plot as upset_plot
import df2img
from IPython.display import display, HTML<jupyter_output><empty_output><jupyter_text>Define function for forcing a table to display all of it contents<jupyter_code>def force_show_all(df):
    with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', None):
        display(HTML(df.to_html()))<jupyter_output><empty_output><jupyter_text>Data preparation: Read gffcmp.tracking from the [GFFcompare](https://ccb.jhu.edu/software/stringtie/gffcompare.shtml) results, adjust lines for missing values and populate dictonary.<jupyter_code>tcons = {}
with open(snakemake.input.tracking) as file:
    for line in file:
        line = line.split()
        if ',' in line[4][3::]:
            first = line[4][3::].split(',')
            q1 = first[0].split('|')
        else:
            q1 = line[4][3::].split('|')
        if ',' in line[5][3::]:
            first = line[5][3::].split(',')
            q2 = first[0].split('|')
        else:
            q2 = line[5][3::].split('|')
        if ',' in line[6][3::]:
            first = line[6][3::].split(',')
            q3 = first[0].split('|')
        else:
            q3 = line[6][3::].split('|')
        refgeneid = line[2].split('|')
        if len(q1) < 2:
            q1 = ['NA', 'NA', 0, 0, 0, 0, 0]
        if len(q2) < 2:
            q2 = ['NA', 'NA', 0, 0, 0, 0, 0]
        if len(q3) < 2:
            q3 = ['NA', 'NA', 0, 0, 0, 0, 0]
        if len(refgeneid) < 2:
            refgeneid = ['NA', 'NA']
        tcons[line[0]] = [line[0], line[1], refgeneid[0], refgeneid[1], line[3],
                          q1[0], q1[1], q1[2], q1[3], q1[4], q1[5], q1[6],
                          q2[0], q2[1], q2[2], q2[3], q2[4], q2[5], q2[6],
                          q3[0], q3[1], q3[2], q3[3], q3[4], q3[5], q3[6]]<jupyter_output><empty_output><jupyter_text>Define column names for dataframe<jupyter_code>column_names = ['Query transfrag id', 'Query locus id', 'Reference gene name', 'Reference gene id', 'Class code',
                'oxford.gene_id', 'oxford.transcript_id', 'oxford.num_exons', 'oxford.FPKM', 'oxford.counts',
                'oxford.cov', 'oxford.len', 'flair.gene_id', 'flair.transcript_id', 'flair.num_exons',
                'flair.FPKM', 'flair.counts', 'flair.cov', 'flair.len', 'talon.gene_id', 'talon.transcript_id',
                'talon.num_exons', 'talon.FPKM', 'talon.counts', 'talon.cov', 'talon.len']<jupyter_output><empty_output><jupyter_text>Convert dictionary to dataframe<jupyter_code>df = pd.DataFrame.from_dict(tcons, orient='index', columns=column_names)<jupyter_output><empty_output><jupyter_text>Assign correct datatypes to dataframe columns<jupyter_code>df = df.astype({'Query transfrag id': str, 'Query locus id': str, 'Reference gene name': str, 'Reference gene id': str,
                'Class code': str, 'oxford.gene_id': str, 'oxford.transcript_id': str, 'oxford.num_exons': int,
                'oxford.FPKM': float, 'oxford.counts': float, 'oxford.cov': float,
                'oxford.len': int, 'flair.gene_id': str, 'flair.transcript_id': str, 'flair.num_exons': int,
                'flair.FPKM': float, 'flair.counts': float, 'flair.cov': float, 'flair.len': int,
                'talon.gene_id': str, 'talon.transcript_id': str, 'talon.num_exons': int, 'talon.FPKM': float,
                'talon.counts': float, 'talon.cov': float, 'talon.len': int})<jupyter_output><empty_output><jupyter_text>Take a look at the dataframe<jupyter_code>df<jupyter_output><empty_output><jupyter_text>Venn diagrams Set matplotlib settings<jupyter_code>plt.rcParams["figure.figsize"] = [10, 10]
plt.rcParams["figure.autolayout"] = False
plt.rcParams.update({'font.size': 15})<jupyter_output><empty_output><jupyter_text>Extract transfragments with a count higher then zero for each pipeline<jupyter_code>ox_mask = (df['oxford.counts'] > 0)
ox_array = df[ox_mask].values
ox_values = ox_array.T[0]

flair_mask = (df['flair.counts'] > 0)
flair_array = df[flair_mask].values
flair_values = flair_array.T[0]

talon_mask = (df['talon.counts'] > 0)
talon_array = df[talon_mask].values
talon_values = talon_array.T[0]<jupyter_output><empty_output><jupyter_text>Create a upset plot that shows overlap of transcripts that are  present in the samples<jupyter_code>transcripts = from_contents({'oxford': set(ox_values), 'flair': set(flair_values), 'talon': set(talon_values)})
upset_plot(transcripts, element_size=50)
plt.suptitle('Overlap of transcripts present')
plt.savefig(snakemake.output.all_upset, bbox_inches='tight', dpi=200)<jupyter_output><empty_output><jupyter_text>Create a venndiagram that shows overlap of transcripts that are  present in the samples<jupyter_code>venn3([set(ox_values), set(flair_values), set(talon_values)], ('oxford', 'flair', 'talon'))
plt.title('Overlap of transcripts present')
plt.savefig(snakemake.output.all_diagram, bbox_inches='tight', dpi=200)<jupyter_output><empty_output><jupyter_text>Extract transfrag's with a count higher then zero and have a '=' class code.<jupyter_code>ox_mask_known = (df['oxford.counts'] > 0) & (df['Class code'] == '=')
ox_array_known = df[ox_mask_known].values
ox_values_known = ox_array_known.T[0]

flair_mask_known = (df['flair.counts'] > 0) & (df['Class code'] == '=')
flair_array_known = df[flair_mask_known].values
flair_values_known = flair_array_known.T[0]

talon_mask_known = (df['talon.counts'] > 0) & (df['Class code'] == '=')
talon_array_known = df[talon_mask_known].values
talon_values_known = talon_array_known.T[0]<jupyter_output><empty_output><jupyter_text>Create a upset plot with present transcripts that have being classified as known by GFFcompare.<jupyter_code>transcripts = from_contents({'oxford': set(ox_values_known), 'flair': set(flair_values_known), 'talon': set(talon_values_known)})
upset_plot(transcripts, element_size=50)
plt.suptitle('Overlap of transcripts with classcode: "="')
plt.savefig(snakemake.output.known_upset, bbox_inches='tight', dpi=200)<jupyter_output><empty_output><jupyter_text>Create a venndiagram with present transcripts that have being classified as known by GFFcompare.<jupyter_code>venn3([set(ox_values_known), set(flair_values_known), set(talon_values_known)], ('oxford', 'flair', 'talon'))
plt.title('Overlap of transcripts with classcode: "="')
plt.savefig(snakemake.output.known_diagram, bbox_inches='tight', dpi=200)<jupyter_output><empty_output><jupyter_text>Extract transfrag's with a count higher then zero and do not have a '=' class code.<jupyter_code>ox_mask_novel = (df['oxford.counts'] > 1) & (df['Class code'] != '=')
ox_array_novel = df[ox_mask_novel].values
ox_values_novel = ox_array_novel.T[0]

flair_mask_novel = (df['flair.counts'] > 1) & (df['Class code'] != '=')
flair_array_novel = df[flair_mask_novel].values
flair_values_novel = flair_array_novel.T[0]

talon_mask_novel = (df['talon.counts'] > 1) & (df['Class code'] != '=')
talon_array_novel = df[talon_mask_novel].values
talon_values_novel = talon_array_novel.T[0]<jupyter_output><empty_output><jupyter_text>Create a venndiagram with present transcripts that have being classified as novel by GFFcompare<jupyter_code>transcripts = from_contents({'oxford': set(ox_values_novel), 'flair': set(flair_values_novel), 'talon': set(talon_values_novel)})
upset_plot(transcripts, element_size=50)
plt.suptitle('Overlap of transcripts that do NOT have classcode: "="')
plt.savefig(snakemake.output.novel_upset, bbox_inches='tight', dpi=200)<jupyter_output><empty_output><jupyter_text>Create a venndiagram with present transcripts that have being classified as novel by GFFcompare<jupyter_code>venn3([set(ox_values_novel), set(flair_values_novel), set(talon_values_novel)], ('oxford', 'flair', 'talon'))
plt.title('Overlap of transcripts that do NOT have classcode: "="')
plt.savefig(snakemake.output.novel_diagram, bbox_inches='tight', dpi=200)<jupyter_output><empty_output><jupyter_text>Non-matched transcripts analyses Get all unique transcripts per pipeline<jupyter_code>oxford_filter_mask = (df['oxford.counts'] > 0) & (df['flair.counts'] == 0) & (df['talon.counts'] == 0)
oxford_single = df[oxford_filter_mask]
oxford_single_df = oxford_single[['Reference gene id','oxford.num_exons', 'oxford.len','oxford.counts']]

flair_filter_mask = (df['oxford.counts'] == 0) & (df['flair.counts'] > 0) & (df['talon.counts'] == 0)
flair_single = df[flair_filter_mask]
flair_single_df = flair_single[['Reference gene id','flair.num_exons', 'flair.len','flair.counts']]

talon_filter_mask = (df['oxford.counts'] == 0) & (df['flair.counts'] == 0) & (df['talon.counts'] > 0)
talon_single = df[talon_filter_mask]
talon_single_df = talon_single[['Reference gene id','talon.num_exons', 'talon.len','talon.counts']]<jupyter_output><empty_output><jupyter_text>Get statistics per pipeline for the lenght of the unique transcripts<jupyter_code>ox_len = oxford_single_df['oxford.len'].describe().to_frame('OXFORD')
fl_len = flair_single_df['flair.len'].describe().to_frame('FLAIR')
ta_len = talon_single_df['talon.len'].describe().to_frame('TALON')
length_frames = [ox_len, fl_len, ta_len]

merged_length = pd.concat(length_frames, axis='columns', verify_integrity=True)
force_show_all(merged_length)<jupyter_output><empty_output><jupyter_text>Get statistics per pipeline for the count of the unique transcripts<jupyter_code>ox_count = oxford_single_df['oxford.counts'].describe().to_frame('OXFORD')
fl_count = flair_single_df['flair.counts'].describe().to_frame('FLAIR')
ta_count = talon_single_df['talon.counts'].describe().to_frame('TALON')
count_frames = [ox_count, fl_count, ta_count]

merged_counts = pd.concat(count_frames, axis='columns', verify_integrity=True)
force_show_all(merged_counts)<jupyter_output><empty_output><jupyter_text>Show transcripts with number of exons<jupyter_code>ox_exons = oxford_single_df['oxford.num_exons'].value_counts()
fl_exons = flair_single_df['flair.num_exons'].value_counts()
ta_exons = talon_single_df['talon.num_exons'].value_counts()

ox_exons = ox_exons.to_frame('OXFORD')
fl_exons = fl_exons.to_frame('FLAIR')
ta_exons = ta_exons.to_frame('TALON')
exon_frames = [ox_exons, fl_exons, ta_exons]

merged_exons = pd.concat(exon_frames, axis='columns', verify_integrity=True)
merged_exons = merged_exons.sort_index(ascending=True)
# force_show_all(merged_exons)
merged_exons.plot(title="None-matched transcripts per number of exons", xlabel="Exons in transcript", ylabel="Number of transcripts (log)", kind='bar', logy=True, figsize=(20,5))<jupyter_output><empty_output><jupyter_text>GffCompare classes Get the GFFcompare classes of ALL transcripts matched against the genome annotation.<jupyter_code>mask = (df['oxford.counts']>0)
oxford = df[mask].groupby('Class code', dropna=False).size()
ox_count = oxford.to_frame('Oxford')

mask = (df['flair.counts']>0)
flair = df[mask].groupby('Class code', dropna=False).size()
flair_count = flair.to_frame('FLAIR')

mask = (df['talon.counts']>0)
talon = df[mask].groupby('Class code', dropna=False).size()
talon_count = talon.to_frame('TALON')

classes = talon_count.merge(flair_count, left_on='Class code', right_on='Class code', how='outer')\
.merge(ox_count, left_on='Class code', right_on='Class code', how='outer')

classes<jupyter_output><empty_output><jupyter_text>Get the GffCompare classes of UNIQUE transcripts matched against the genome annotation.<jupyter_code>mask = (df['oxford.counts']>0) & (df['flair.counts'] == 0) & (df['talon.counts'] == 0)
oxford = df[mask].groupby('Class code', dropna=False).size()
ox_count = oxford.to_frame('Oxford')

mask = (df['flair.counts']>0) & (df['oxford.counts'] == 0) & (df['talon.counts'] == 0)
flair = df[mask].groupby('Class code', dropna=False).size()
flair_count = flair.to_frame('FLAIR')

mask = (df['talon.counts']>0) & (df['flair.counts'] == 0) & (df['oxford.counts'] == 0)
talon = df[mask].groupby('Class code', dropna=False).size()
talon_count = talon.to_frame('TALON')

classes = talon_count.merge(flair_count, left_on='Class code', right_on='Class code', how='outer')\
.merge(ox_count, left_on='Class code', right_on='Class code', how='outer')

classes<jupyter_output><empty_output><jupyter_text>Get the GffCompare classes of three way matches<jupyter_code>mask = (df['oxford.counts']>0) & (df['flair.counts']>0) & (df['talon.counts']>0)
three = df[mask].groupby('Class code', dropna=False).size()
three_count = three.to_frame('Three match')
three_count<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1598483651871742, 'fraction_numerical': 0.014294740167430106, 'mean_word_length': 6.074301675977654, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 118, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '145256', 'license': 'MIT', 'n_tokens_mistral': 4916, 'n_tokens_neox': 4650, 'n_words': 847, 'path': 'workflow/notebooks/venndiagrams.py.ipynb', 'repo_name': 'bayraktar1/RPCA', 'size': '12662'}