<commit_before>"""
Modules to support sqlite implementation
"""
import os.path as op
import sqlite3 as lite
import json

from bcbio.utils import safe_makedir


def _create_db(name):
    """
    creater connection to sqlite
    """
    con = lite.connect(name)
    return con


def _insert_data(con, data):
    """
    insert line for each cluster
    """
    with con:
        cur = con.cursor()
        cur.execute("DROP TABLE IF EXISTS clusters;")
        cur.execute("CREATE TABLE clusters(Id INT, Locus TEXT, Annotation TEXT, Sequences TEXT)")
        for c in data[0]:
            locus = json.dumps(data[0][c]['loci'])
            annotation = json.dumps(data[0][c]['ann'])
            sequences = json.dumps(data[0][c]['seqs'])
            cur.execute("INSERT INTO clusters VALUES(%s, '%s', '%s', '%s')" % (c, locus, annotation, sequences))


def _close(con):
    if con:
        con.close()


def make_database(data, name="seqcluster.db", out_dir="database"):
    out_dir = safe_makedir(out_dir)
    op.abspath(out_dir)
    con = _create_db(op.join(out_dir, name))
    _insert_data(con, data)
    _close(con)
<commit_msg>add description to database<commit_after>"""
Modules to support sqlite implementation
"""
import os.path as op
import sqlite3 as lite
import json

from bcbio.utils import safe_makedir


def _create_db(name):
    """
    creater connection to sqlite
    """
    con = lite.connect(name)
    return con


def _get_description(string):
    """
    Parse annotation to get nice description
    """
    ann = set()
    if not string:
        return "This cluster is inter-genic."
    for item in string:
        for db in item:
            ann = ann.union(set(item[db]))
    return "annotated as: %s ..." % ",".join(list(ann)[:3])

def _insert_data(con, data):
    """
    insert line for each cluster
    """
    with con:
        cur = con.cursor()
        cur.execute("DROP TABLE IF EXISTS clusters;")
        cur.execute("CREATE TABLE clusters(Id INT, Description TEXT, Locus TEXT, Annotation TEXT, Sequences TEXT)")
        for c in data[0]:
            locus = json.dumps(data[0][c]['loci'])
            annotation = json.dumps(data[0][c]['ann'])
            description = _get_description(data[0][c]['ann'])
            print description
            sequences = json.dumps(data[0][c]['seqs'])
            cur.execute("INSERT INTO clusters VALUES(%s, '%s', '%s', '%s', '%s')" % (c, description, locus, annotation, sequences))


def _close(con):
    if con:
        con.close()


def make_database(data, name="seqcluster.db", out_dir="database"):
    out_dir = safe_makedir(out_dir)
    op.abspath(out_dir)
    con = _create_db(op.join(out_dir, name))
    _insert_data(con, data)
    _close(con)

{'fraction_non_alphanumeric': 0.138348623853211, 'fraction_numerical': 0.004403669724770643, 'mean_word_length': 2.894285714285714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '88041', 'n_tokens_mistral': 896, 'n_tokens_neox': 870, 'n_words': 236}