<jupyter_start><jupyter_text>Merge all the dfs that were made- So all the checks we did to see if a contig/protien was viral- RdRp hmm matches- Vibrant output- Vibrant text search output- BlastP output- BlastN output (only one per contig)<jupyter_code># imports
import pandas as pd
import numpy as np
# open the RdRp search results
df_rdrp =  open_df('201228_RdRp_hits_eval_adapt.csv', ',')

# open vibrant results
df_vibrant = open_df('VIBRANT_annotations_all_contigs.txt','\t')

# open vibrant text search
df_vibrant_text = open_df('210119_viral_proteins_found_vibrant.csv', ',')

# open blastp
df_blastp = open_df('210120_blastp_output_w_desc_selectedviral.csv', ',')

# open blastn
df_blastn= open_df('210119_blastn_output_w_desc.csv', ',')
# give df_rdrp a column with contig names
df_rdrp['contig'] = df_rdrp.protein.str[:-2]

# give an extra column where we remove seed from the pfam number
df_rdrp['pfam_num'] = df_rdrp['hit'].str.replace('_seed','')

#Open df with the PFAM name -number combinations
df_pfam_names = open_df('rdrp_pfam_namenumber.csv', ',')

# replace the PFAM numbers in this df with the RdRp names
# write names and numbers to dict
pfam_dict = df_pfam_names.set_index('rdrp_name')['pfam_num'].to_dict()

# replace
df_rdrp['pfam_num'] = df_rdrp['pfam_num'].apply(str).replace(pfam_dict, regex=False)

# print head new df
df_rdrp.head()
# rename vibrant column to contig
df_vibrant = df_vibrant.rename(columns={'scaffold': 'contig'})


# same for blastn
df_blastn = df_blastn.rename(columns={'qseqid': 'contig'})

# same for blastp
df_blastp = df_blastp.rename(columns={'qseqid': 'protein'})
# concatenate all the protein outpyut into one file
df_all_protein = df_rdrp.merge(df_vibrant,on='protein',how='outer').merge(df_vibrant_text,on='protein',how='outer').merge(df_blastp,on='protein',how='outer')

# make a new column contig so that we can merge with blastN results
df_all_protein['contig'] = df_all_protein.protein.str[:-2]

# merge protein results with blastn
df_all = df_all_protein.merge(df_blastn,on='contig',how='outer')

# write to file
df_all.to_csv('210120_all_resultsconcat.csv')
len(df_blastn_tocheck)
df_blastn_tocheck = df_all[~df_all.description_blastn.str.contains("virus", na=False) & ~df_all.description_blastp.str.contains("virus", na=False)]

# Write to file for manual check
df_blastn_tocheck.to_csv('210120_stringent_check.csv')
df_less_check = df_all[df_all.description_blastp.str.contains("virus", na=False) | df_all.description_blastn.str.contains("virus", na=False)]
# Write to file for manual check
df_less_check.to_csv('210120_less_check.csv')
def open_df(df, sep):
    # open the df
    df = pd.read_csv(df, sep=sep)
    # show top
    display(df.head())
    return df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12336719883889695, 'fraction_numerical': 0.015965166908563134, 'mean_word_length': 6.512261580381471, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '538196', 'license': 'MIT', 'n_tokens_mistral': 1105, 'n_tokens_neox': 1034, 'n_words': 228, 'path': '210120_concat_alloutputs.ipynb', 'repo_name': 'AnneliektH/arboretum', 'size': '2756'}