<commit_before>
# DB INIT PY
# Initialize the SQLite DB
# See db-init.sql for the table schema

from xcorr_db import xcorr_db, q

DB = xcorr_db('xcorr.db')

def create_tables():
    """ Set up the tables defined in the SQL file """
    with open("db-init.sql") as fp:
        sqlcode = fp.read()
    DB.executescript(sqlcode);
    DB.commit()

def insert_feature_names():
    """
    Copy features from the header of this datafile
    into the features table
    """
    datafile = "test_data/combined_rnaseq_data_lincs1000_combat"

    with open(datafile) as fp:
        line = fp.readline()

    feature_names = line.split("\t")
    del feature_names[0] # Remove first token "Sample"

    for name in feature_names:
        if name == "": continue
        name = name.strip()
        DB.insert("feature_names", ["name"], [q(name)])

def insert_study_names():
    """ Copy study names from studies.txt into the DB """
    studies = []
    with open("studies.txt") as fp:
        while True:
            line = fp.readline()
            if line == "": break
            tokens = line.split("#")
            line = tokens[0]
            line = line.strip()
            if line == "": continue
            studies.append(line)

    for study in studies:
        DB.insert("study_names", ["name"], [q(study)])

def create_indices():
    """ Create indices after data insertion for speed """
    DB.execute("create index features_index on features(record_id);")
    DB.execute("create index  studies_index on studies ( study_id);")

# Catch and print all exceptions to improve visibility of success/failure
success = False
try:
    create_tables()
    insert_feature_names()
    insert_study_names()
    create_indices()
    success = True
except Exception as e:
    import traceback
    print(traceback.format_exc())

if not success:
    print("DB: !!! INIT FAILED !!!")
    import sys
    sys.exit(1)

print("DB: initialized successfully")
<commit_msg>More self-doc<commit_after>
# DB INIT PY
# Initialize the SQLite DB
# See db-init.sql for the table schema

from xcorr_db import xcorr_db, q

DB = xcorr_db('xcorr.db')

def create_tables():
    """ Set up the tables defined in the SQL file """
    with open("db-init.sql") as fp:
        sqlcode = fp.read()
    DB.executescript(sqlcode);
    DB.commit()

def insert_feature_names():
    """
    Copy features from the header of this datafile
    into the features table
    """
    datafile = "test_data/combined_rnaseq_data_lincs1000_combat"

    with open(datafile) as fp:
        line = fp.readline()

    feature_names = line.split("\t")
    del feature_names[0] # Remove first token "Sample"

    for name in feature_names:
        if name == "": continue
        name = name.strip()
        DB.insert(table="feature_names",
                  names=["name"],
                  values=[q(name)])

def insert_study_names():
    """ Copy study names from studies.txt into the DB """
    studies = []
    with open("studies.txt") as fp:
        while True:
            line = fp.readline()
            if line == "": break
            tokens = line.split("#")
            line = tokens[0]
            line = line.strip()
            if line == "": continue
            studies.append(line)

    for study in studies:
        DB.insert(table="study_names",
                  names=["name"],
                  values=[q(study)])

def create_indices():
    """ Create indices after data insertion for speed """
    DB.execute("create index features_index on features(record_id);")
    DB.execute("create index  studies_index on studies ( study_id);")

# Catch and print all exceptions to improve visibility of success/failure
success = False
try:
    create_tables()
    insert_feature_names()
    insert_study_names()
    create_indices()
    success = True
except Exception as e:
    import traceback
    print(traceback.format_exc())

if not success:
    print("DB: !!! INIT FAILED !!!")
    import sys
    sys.exit(1)

print("DB: initialized successfully")

{'fraction_non_alphanumeric': 0.11645885286783042, 'fraction_numerical': 0.003491271820448878, 'mean_word_length': 2.629864253393665, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '785432', 'n_tokens_mistral': 1220, 'n_tokens_neox': 1189, 'n_words': 359}