<commit_before>#database.py creates a .db file for performing umls searches.
import sqlite3
import os
import sys
import os

def create_db():

    print "\ncreating umls.db"
    #connect to the .db file we are creating.
    db_path = os.path.join(os.environ['CLINER_DIR'],'umls_tables/umls.db')
    conn = sqlite3.connect( db_path )
    conn.text_factory = str

    print "opening files"
    #load data in files.
    try:
        mrsty_path = os.path.join(os.environ['CLINER_DIR'],'umls_tables/MRSTY')
        MRSTY_TABLE_FILE = open( mrsty_path, "r" )
    except IOError:
        print "\nNo file to use for creating MRSTY table\n"
        conn.close()
        sys.exit()

    try:
        mrcon_path = os.path.join(os.environ['CLINER_DIR'],'umls_tables/MRCON')
        MRCON_TABLE_FILE = open( mrcon_path , "r" )
    except IOError:
        print "\nNo file to use for creating MRCON table\n"
        conn.close()
        sys.exit()

    try:
        mrrel_path = os.path.join(os.environ['CLINER_DIR'],'umls_tables/MRREL')
        MRREL_TABLE_FILE = open( mrrel_path , "r" )
    except IOError:
        print "\nNo file to use for creating MRREL table\n"
        conn.close()
        sys.exit()

    print "creating tables"
    c = conn.cursor()

    #create tables.
    c.execute( "CREATE TABLE MRCON( CUI, LAT, TS, LUI, STT, SUI, STR, LRL, EMPTY ) ;" )
    c.execute( "CREATE TABLE MRSTY( CUI, TUI, STY, EMPTY ) ;" )
    c.execute( "CREATE TABLE MRREL( CUI1, REL, CUI2, RELA, SAB, SL, MG, EMPTY ) ;" )

    print "inserting data into MRSTY table"
    for line in MRSTY_TABLE_FILE:

        try:
            c.execute( "INSERT INTO MRCON( CUI, LAT, TS, LUI, STT, SUI, STR, LRL, EMPTY ) values ( ?, ?, ? ,?, ?,?,?,?,?);", tuple(line[0:-1].split('|')) )
        except sqlite3.ProgrammingError:
            continue

    MRSTY_TABLE_FILE.close()

    print "inserting data into MRCON table"
    for line in MRCON_TABLE_FILE:

        try:
            c.execute( "INSERT INTO MRSTY( CUI, TUI, STY, EMPTY) values( ?, ?, ?, ?)" , tuple(line[0:-1].split('|')) )
        except sqlite3.ProgrammingError:
            continue<commit_msg>fixed error, was inserting wrong data into tables<commit_after>#database.py creates a .db file for performing umls searches.
import sqlite3
import os
import sys
import os

def create_db():

    print "\ncreating umls.db"
    #connect to the .db file we are creating.
    db_path = os.path.join(os.environ['CLINER_DIR'],'umls_tables/umls.db')
    conn = sqlite3.connect( db_path )
    conn.text_factory = str

    print "opening files"
    #load data in files.
    try:
        mrsty_path = os.path.join(os.environ['CLINER_DIR'],'umls_tables/MRSTY')
        MRSTY_TABLE_FILE = open( mrsty_path, "r" )
    except IOError:
        print "\nNo file to use for creating MRSTY table\n"
        conn.close()
        sys.exit()

    try:
        mrcon_path = os.path.join(os.environ['CLINER_DIR'],'umls_tables/MRCON')
        MRCON_TABLE_FILE = open( mrcon_path , "r" )
    except IOError:
        print "\nNo file to use for creating MRCON table\n"
        conn.close()
        sys.exit()

    try:
        mrrel_path = os.path.join(os.environ['CLINER_DIR'],'umls_tables/MRREL')
        MRREL_TABLE_FILE = open( mrrel_path , "r" )
    except IOError:
        print "\nNo file to use for creating MRREL table\n"
        conn.close()
        sys.exit()

    print "creating tables"
    c = conn.cursor()

    #create tables.
    c.execute( "CREATE TABLE MRCON( CUI, LAT, TS, LUI, STT, SUI, STR, LRL, EMPTY ) ;" )
    c.execute( "CREATE TABLE MRSTY( CUI, TUI, STY, EMPTY ) ;" )
    c.execute( "CREATE TABLE MRREL( CUI1, REL, CUI2, RELA, SAB, SL, MG, EMPTY ) ;" )

    print "inserting data into MRSTY table"
    for line in MRSTY_TABLE_FILE:

        try:
            c.execute( "INSERT INTO MRSTY( CUI, TUI, STY, EMPTY) values( ?, ?, ?, ?)" , tuple(line[0:-1].split('|')) )
        except sqlite3.ProgrammingError:
            continue

    MRSTY_TABLE_FILE.close()

    print "inserting data into MRCON table"
    for line in MRCON_TABLE_FILE:

        try:
            c.execute( "INSERT INTO MRCON( CUI, LAT, TS, LUI, STT, SUI, STR, LRL, EMPTY ) values ( ?, ?, ? ,?, ?,?,?,?,?);", tuple(line[0:-1].split('|')) )
        except sqlite3.ProgrammingError:
            continue
{'fraction_non_alphanumeric': 0.13192182410423453, 'fraction_numerical': 0.004653327128897162, 'mean_word_length': 2.576539101497504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7142396', 'n_tokens_mistral': 1592, 'n_tokens_neox': 1536, 'n_words': 402}