<jupyter_start><jupyter_text>Remove random  lines from fasta<jupyter_code>five_prime_cleaned_fasta = '/panfs/qcb-panasas/skchoudh/genomes/R64-1-1/annotation/SGD_all_ORFs/SGD_all_ORFs_5prime_UTRs_cleaned.fsa'
three_prime_cleaned_fasta = '/panfs/qcb-panasas/skchoudh/genomes/R64-1-1/annotation/SGD_all_ORFs/SGD_all_ORFs_3prime_UTRs_cleaned.fsa'

lines = []
with open(five_prime_fasta) as fh:
    for line in fh:
        if line[0]!="#":
            lines.append(line)
with open(five_prime_cleaned_fasta, 'w') as fh:
    fh.writelines(lines)
lines = []
with open(three_prime_fasta) as fh:
    for line in fh:
        if line[0]!="#":
            lines.append(line)
with open(three_prime_cleaned_fasta, 'w') as fh:
    fh.writelines(lines)
five_prime_idx = pyfaidx.Fasta(five_prime_cleaned_fasta)
three_prime_idx = pyfaidx.Fasta(three_prime_cleaned_fasta)
from Bio import SeqIO
five_prime_records = []
for record in SeqIO.parse(five_prime_cleaned_fasta, "fasta"):
    id = record.id
    description = record.description.replace(id, '').strip()
    chrom_start_stop = description.split(" 5'pad")[0].replace('range=', '')
    chrom = chrom_start_stop.split(':')[0].replace('chr', '')
    start,stop = chrom_start_stop.split('-')
    stop = int(stop)
    start = start.split(':')[1]
    start = int(start)
    strand = description.split('strand=')[1][0]
    five_pad = description.split("5'pad")[1].split(" ")[0]
    three_pad = description.split("3'pad")[1].split(" ")[0]
    
    gene_name = id.split('_')[4]
    source = ('_').join(id.split('_')[8:10])
    five_prime_records.append((chrom, source, 'five_prime_utr', start, stop, '.', strand, '.',  'gene_id "{}"; transcript_id "{}_mRNA"; gene_source "{}"; gene_biotype "protein_coding"; transcript_biotype "protein_coding";'.format(gene_name, gene_name, source)))
    five_prime_records.append((chrom, source, 'exon', start, stop, '.', strand, '.',  'gene_id "{}"; transcript_id "{}_mRNA"; gene_source "{}"; gene_biotype "protein_coding"; transcript_biotype "protein_coding";'.format(gene_name, gene_name, source)))
five_prime_records_df = pd.DataFrame(five_prime_records).drop_duplicates()
from Bio import SeqIO
three_prime_records = []
for record in SeqIO.parse(three_prime_cleaned_fasta, "fasta"):
    id = record.id
    description = record.description.replace(id, '').strip()
    chrom_start_stop = description.split(" 5'pad")[0].replace('range=', '')
    chrom = chrom_start_stop.split(':')[0].replace('chr', '')
    start,stop = chrom_start_stop.split('-')
    stop = int(stop)
    start = start.split(':')[1]
    start = int(start)
    strand = description.split('strand=')[1][0]
    three_pad = description.split("5'pad")[1].split(" ")[0]
    three_pad = description.split("3'pad")[1].split(" ")[0]
    
    gene_name = id.split('_')[4]
    source = ('_').join(id.split('_')[8:10])
    three_prime_records.append((chrom, source, 'three_prime_utr', start, stop, '.', strand, '.',  'gene_id "{}"; transcript_id "{}_mRNA"; gene_source "{}"; gene_biotype "protein_coding"; transcript_biotype "protein_coding";'.format(gene_name, gene_name, source)))
    three_prime_records.append((chrom, source, 'exon', start, stop, '.', strand, '.',  'gene_id "{}"; transcript_id "{}_mRNA"; gene_source "{}"; gene_biotype "protein_coding"; transcript_biotype "protein_coding";'.format(gene_name, gene_name, source)))
three_prime_records_df = pd.DataFrame(three_prime_records).drop_duplicates()

three_prime_records_df
os.path.basename(original_gtf)<jupyter_output><empty_output><jupyter_text>Merge the GTF<jupyter_code>original_gtf = '/panfs/qcb-panasas/skchoudh/genomes/R64-1-1/annotation/Saccharomyces_cerevisiae.R64-1-1.96.gtf'
original_gtf_lines = []
with open(original_gtf) as fh:
    for line in fh:
        original_gtf_lines.append(line)
original_gtf_df = pd.read_csv(original_gtf, sep="\t", skiprows=5, header=None)
original_gtf_df.head()
gtf_df_combined = pd.concat([original_gtf_df, five_prime_records_df, three_prime_records_df])
gtf_df_combined = gtf_df_combined.sort_values(by=[0, 3,4,2, 6])
gtf_df_combined.to_csv("/panfs/qcb-panasas/skchoudh/genomes/R64-1-1/annotation/Saccharomyces_cerevisiae.R64-1-1.96.combined.UTR5UTR3.gtf", sep="\t", index=False, header=False) 
gtf_df_combined.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16779780014041656, 'fraction_numerical': 0.016849988298619237, 'mean_word_length': 5.730708661417323, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '656960', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1701, 'n_tokens_neox': 1617, 'n_words': 230, 'path': 'notebooks/R64-1-1-get_UTRs.ipynb', 'repo_name': 'anamika-yadav99/gencode_regions', 'size': '4273'}