<commit_before># usage: python build-db.py <wod ascii file name> <table name to append to>

try:
    import psycopg2 as db
    dbtype = 'postgres'
    concom = "dbname='root' user='root'"
except:
    import sqlite3 as db
    concom = 'qcresults.sqlite'
    dbtype = 'sqlite'
print('Database type is ' + dbtype)
from wodpy import wod
import sys
import util.main as main

if len(sys.argv) == 3:

    # connect to database and create a cursor by which to interact with it.
    try:
        conn = db.connect(concom)
    except:
        print "I am unable to connect to the database"

    cur = conn.cursor()

    # Identify tests
    testNames = main.importQC('qctests')
    testNames.sort()

    # set up our table
    query = "CREATE TABLE IF NOT EXISTS " + sys.argv[2] + """(
                raw text,
                truth integer,
                uid integer,
                year integer,
                month integer,
                lat real, 
                long real, 
                cruise integer,
                """
    for i in range(len(testNames)):
        query += testNames[i].lower() + ' integer'
        if i<len(testNames)-1:
            query += ','
        else:
            query += ');'

    cur.execute(query)

    # populate table from wod-ascii data
    fid = open(sys.argv[1])
    while True:
        # extract profile as wodpy object and raw text
        start = fid.tell()
        profile = wod.WodProfile(fid)
        end = fid.tell()
        fid.seek(start)
        raw = fid.read(end-start)
        fid.seek(end)

        # set up dictionary for populating query string
        wodDict = profile.npdict()
        wodDict['raw'] = "'" + raw + "'"
        wodDict['truth'] = int(sum(profile.t_level_qc(originator=True) >= 3) >= 1)
    
        query = "INSERT INTO " + sys.argv[2] + " (raw, truth, uid, year, month, lat, long, cruise) "  + """ VALUES(
                    {p[raw]},
                    {p[truth]},
                    {p[uid]},
                    {p[year]},
                    {p[month]},
                    {p[latitude]}, 
                    {p[longitude]}, 
                    {p[cruise]}
                   )""".format(p=wodDict)
        query = query.replace('--', 'NULL')
        query = query.replace('None', 'NULL')
        cur.execute(query)
        if profile.is_last_profile_in_file(fid) == True:
            break

    conn.commit()

else:<commit_msg>Catch problem with missing data when creating the database.<commit_after># usage: python build-db.py <wod ascii file name> <table name to append to>

try:
    import psycopg2 as db
    dbtype = 'postgres'
    concom = "dbname='root' user='root'"
except:
    import sqlite3 as db
    concom = 'qcresults.sqlite'
    dbtype = 'sqlite'
print('Database type is ' + dbtype)
from wodpy import wod
import sys
import util.main as main

if len(sys.argv) == 3:

    # connect to database and create a cursor by which to interact with it.
    try:
        conn = db.connect(concom)
    except:
        print "I am unable to connect to the database"

    cur = conn.cursor()

    # Identify tests
    testNames = main.importQC('qctests')
    testNames.sort()

    # set up our table
    query = "CREATE TABLE IF NOT EXISTS " + sys.argv[2] + """(
                raw text,
                truth integer,
                uid integer,
                year integer,
                month integer,
                lat real, 
                long real, 
                cruise integer,
                """
    for i in range(len(testNames)):
        query += testNames[i].lower() + ' integer'
        if i<len(testNames)-1:
            query += ','
        else:
            query += ');'

    cur.execute(query)

    # populate table from wod-ascii data
    fid = open(sys.argv[1])
    while True:
        # extract profile as wodpy object and raw text
        start = fid.tell()
        profile = wod.WodProfile(fid)
        end = fid.tell()
        fid.seek(start)
        raw = fid.read(end-start)
        fid.seek(end)

        # set up dictionary for populating query string
        wodDict = profile.npdict()
        wodDict['raw'] = "'" + raw + "'"
        # Below avoids failures if all profile data are missing.
        # We have no use for this profile in that case so skip it.
        try:
            wodDict['truth'] = int(sum(profile.t_level_qc(originator=True) >= 3) >= 1)
        except:
            if profile.is_last_profile_in_file(fid) == True:
                break
            continue

        query = "INSERT INTO " + sys.argv[2] + " (raw, truth, uid, year, month, lat, long, cruise) "  + """ VALUES(
                    {p[raw]},
                    {p[truth]},
                    {p[uid]},
                    {p[year]},
                    {p[month]},
                    {p[latitude]}, 
                    {p[longitude]}, 
                    {p[cruise]}
                   )""".format(p=wodDict)
        query = query.replace('--', 'NULL')
        query = query.replace('None', 'NULL')
        cur.execute(query)
        if profile.is_last_profile_in_file(fid) == True:
            break

    conn.commit()

else:
{'fraction_non_alphanumeric': 0.11880225549290298, 'fraction_numerical': 0.00349990278047832, 'mean_word_length': 1.6338965693804404, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '558096', 'n_tokens_mistral': 1610, 'n_tokens_neox': 1524, 'n_words': 434}