<commit_before>#!/usr/bin/env python

import sqlite3
import re
from feature_tbl_entry import FeatureTblEntry

def regexp(expr, item):
    reg = re.compile(expr)
    return reg.search(item) is not None

# Takes blast hit and returns it as a CSV of key=value
def parse_blast_hit_gene(blast_hit):
    if len(blast_hit) == 0 or blast_hit == '.':
        return ''

    attributes = ''
    parts = blast_hit.split('`')

    part1_split = parts[0].split('|')
    attributes += "gene="+re.sub(r'_(.*)$', '', part1_split[2])

    return attributes

# Takes blast hit and returns it as a CSV of key=value
def parse_blast_hit_cds(blast_hit):
    if len(blast_hit) == 0 or blast_hit == '.':
        return 'product=hypothetical protein'

    attributes = ''
    parts = blast_hit.split('`')

    matchObj = re.search( r'RecName: Full=([^;]*)', blast_hit)
    if matchObj:
        attributes += "product="+matchObj.group(1)
    else:
        attributes += "product=hypothetical protein"

    return attributes

# Takes gene ontology and returns it as a CSV of key=value
def parse_gene_ontology(gene_ontology):
    if len(gene_ontology) == 0 or gene_ontology == '.':
        return ''

    attributes = ''
    elements = gene_ontology.split('`')
    for element in elements:
        if len(attributes) > 0:
            attributes += ','

        vals = element.split('^')
        if vals[1] == 'molecular_function':
            attributes += 'go_function='
        elif vals[1] == 'cellular_component':
            attributes += 'go_component='
        elif vals[1] == 'biological_process':
            attributes += 'go_process='
        attributes += vals[2]+'|'+vals[0][3:]+'||IEA'
    return attributes

def reverse_indices(row):
    tmp = row[0]
    row[0] = row[1]
    row[1] = tmp
    return row


class FeatureTblWriter:

    def write_to_db(self, db_conn):

        db_conn.create_function('regexp', 2, regexp)

        # Get the database cursor
        db_cur = db_conn.cursor()

        # Create a new table for our tbl file
        db_cur.execute('CREATE TABLE tbl(prot_id TEXT, seq_id TEXT, type TEXT, starts TEXT, stops TEXT, has_start INT, has_stop INT, strand TEXT, phase TEXT, annotations TEXT)')

        # Select all of the mRNA entries from the gff table
        try:
            db_cur.execute('SELECT * FROM gff WHERE type="mRNA"')
        except OperationalError:
            print("ERROR: gff table doesn't exist in database "+db_name)
            return

        # Iterate through the mRNA
        rnas = db_cur.fetchall()
        for rna in rnas:
            db_cur.execute('SELECT * FROM gff WHERE type="gene" AND id=? LIMIT 1', [rna[10]])
            gene = db_cur.fetchone()

            db_cur.execute('SELECT EXISTS(SELECT prot_id FROM tbl WHERE prot_id=? LIMIT 1)', [rna[9]]) # Make sure this gene entry doesn't already exist
            exists = db_cur.fetchone()[0]
            if exists == 1:
                print("ERROR: Gene already exists: ", rna[9])
                continue
            
            # Get the trinotate stuff
            # TODO try-except here?
            db_cur.execute('SELECT * FROM trinotate WHERE prot_id=? LIMIT 1', [rna[9]])
            trinotate = db_cur.fetchone()

            # Get whether the sequence has start and stop codons
            db_cur.execute('SELECT EXISTS(SELECT type, name FROM gff WHERE type="start_codon" AND name REGEXP ? LIMIT 1)', [rna[9]])
            has_start = db_cur.fetchone()[0]
            db_cur.execute('SELECT EXISTS(SELECT type, name FROM gff WHERE type="stop_codon" AND name REGEXP ? LIMIT 1)', [rna[9]])
            has_stop = db_cur.fetchone()[0]

            # Add the gene entry to our table
            db_cur.execute('INSERT INTO tbl VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [rna[9], gene[1], 'gene', gene[4], gene[5], has_start, has_stop, gene[7], gene[8], ''])

            # Fetch all of the exons pertaining to this gene
            db_cur.execute('SELECT * FROM gff WHERE type="exon" AND name REGEXP ?', [rna[9]])
            exons = db_cur.fetchall()
            if len(exons) > 0: # Make sure there is at least one exon entry
                exon_starts = ''
                exon_stops = ''
                for exon in exons:
                    if len(exon_starts) > 0:
                        exon_starts += ','
                        exon_stops += ','
                    exon_starts += str(exon[4])
                    exon_stops += str(exon[5])

                # Add the exon entry to our table
                db_cur.execute('INSERT INTO tbl VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [rna[9], gene[1], 'exon', exon_starts, exon_stops, has_start, has_stop, gene[7], gene[8], ''])

            # Fetch all of the CDS pertaining to this gene
            db_cur.execute('SELECT * FROM gff WHERE type="CDS" AND name REGEXP ?', [rna[9]])
            cdss = db_cur.fetchall()
            if len(cdss) > 0: # Make sure there is at least one cds entry
                cds_starts = ''
                cds_stops = ''
                for cds in cdss:
                    if len(cds_starts) > 0:
                        cds_starts += ','
                        cds_stops += ','
                    cds_starts += str(cds[4])
                    cds_stops += str(cds[5])

                # Add the CDS entry to our table
                cds_ann = parse_gene_ontology(trinotate[8])
                db_cur.execute('INSERT INTO tbl VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [rna[9], gene[1], 'CDS', cds_starts, cds_stops, has_start, has_stop, gene[7], gene[8], cds_ann])



    def write_to_file(self, db_conn, fileName):<commit_msg>removed FeatureTblWriter.write_to_db() b/c no need, right?<commit_after>#!/usr/bin/env python

import sqlite3
import re
from feature_tbl_entry import FeatureTblEntry

def regexp(expr, item):
    reg = re.compile(expr)
    return reg.search(item) is not None

# Takes blast hit and returns it as a CSV of key=value
def parse_blast_hit_gene(blast_hit):
    if len(blast_hit) == 0 or blast_hit == '.':
        return ''

    attributes = ''
    parts = blast_hit.split('`')

    part1_split = parts[0].split('|')
    attributes += "gene="+re.sub(r'_(.*)$', '', part1_split[2])

    return attributes

# Takes blast hit and returns it as a CSV of key=value
def parse_blast_hit_cds(blast_hit):
    if len(blast_hit) == 0 or blast_hit == '.':
        return 'product=hypothetical protein'

    attributes = ''
    parts = blast_hit.split('`')

    matchObj = re.search( r'RecName: Full=([^;]*)', blast_hit)
    if matchObj:
        attributes += "product="+matchObj.group(1)
    else:
        attributes += "product=hypothetical protein"

    return attributes

# Takes gene ontology and returns it as a CSV of key=value
def parse_gene_ontology(gene_ontology):
    if len(gene_ontology) == 0 or gene_ontology == '.':
        return ''

    attributes = ''
    elements = gene_ontology.split('`')
    for element in elements:
        if len(attributes) > 0:
            attributes += ','

        vals = element.split('^')
        if vals[1] == 'molecular_function':
            attributes += 'go_function='
        elif vals[1] == 'cellular_component':
            attributes += 'go_component='
        elif vals[1] == 'biological_process':
            attributes += 'go_process='
        attributes += vals[2]+'|'+vals[0][3:]+'||IEA'
    return attributes

def reverse_indices(row):
    tmp = row[0]
    row[0] = row[1]
    row[1] = tmp
    return row


class FeatureTblWriter:

    def write_to_file(self, db_conn, fileName):
{'fraction_non_alphanumeric': 0.12917771883289125, 'fraction_numerical': 0.010875331564986738, 'mean_word_length': 2.3382027445772464, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6848068', 'n_tokens_mistral': 2516, 'n_tokens_neox': 2364, 'n_words': 649}