<commit_before>import psycopg2 as pg

class conncur:
    def __init__(self, *args, **kwargs):
        self.conn = pg.connect(*args, **kwargs)
        self.cur = self.conn.cursor()
    def __enter__(self):
        return self.conn, self.cur
    def __exit__(self, type, value, traceback):
        self.cur.close()
        self.conn.close()


def setup_db(create=False):
    """ execute blocks of sql delimited by --words-- in the setup file
        first 4 blocks do user and database setup
        the following 3 blocks do schema and table creation and alters
    """
    with open('heatmap_db_setup.sql', 'rt') as f:
        lines = [' '+l.rstrip('\n').strip(' ') for l in f.readlines()]
    text = ''.join(lines)
    sql_blocks = [l.strip(' ') for l in text.split('--')][::2][1:]  #user, alter user, drop, db, tables, alter

    if create:
        with conncur(dbname='postgres',user='postgres', host='localhost', port=5432) as (conn, cur):
            for sql in sql_blocks[:4]:
                print(sql)
                if sql.startswith('DROP DATABASE') or sql.startswith('CREATE DATABASE') :
                    conn.set_isolation_level(0)
                    cur.execute(sql)
                    conn.commit()
                    conn.set_isolation_level(1)
                else:
                    cur.execute(sql)
                    conn.commit()

            with conncur(dbname='heatmap_test',user='postgres', host='localhost', port=5432) as (conn, cur):
                sql = sql_blocks[4]
                cur.execute(sql)
                conn.commit()

    with conncur(dbname='heatmap_test',user='heatmapadmin', host='localhost', port=5432) as (conn, cur):
        for sql in sql_blocks[5:8]:
            print(sql)
            cur.execute(sql)
            conn.commit()

def main():
    setup_db()

if __name__ == '__main__':
    main()<commit_msg>changed localhost to 127.0.0.1 in dbsetup in the even that postgrest/hosts are configured strangely<commit_after>import psycopg2 as pg

host = '127.0.0.1'

class conncur:
    def __init__(self, *args, **kwargs):
        self.conn = pg.connect(*args, **kwargs)
        self.cur = self.conn.cursor()
    def __enter__(self):
        return self.conn, self.cur
    def __exit__(self, type, value, traceback):
        self.cur.close()
        self.conn.close()


def setup_db(create=False):
    """ execute blocks of sql delimited by --words-- in the setup file
        first 4 blocks do user and database setup
        the following 3 blocks do schema and table creation and alters
    """
    with open('heatmap_db_setup.sql', 'rt') as f:
        lines = [' '+l.rstrip('\n').strip(' ') for l in f.readlines()]
    text = ''.join(lines)
    sql_blocks = [l.strip(' ') for l in text.split('--')][::2][1:]  #user, alter user, drop, db, tables, alter

    if create:
        with conncur(dbname='postgres',user='postgres', host='127.0.0.1', port=5432) as (conn, cur):
            for sql in sql_blocks[:4]:
                print(sql)
                if sql.startswith('DROP DATABASE') or sql.startswith('CREATE DATABASE') :
                    conn.set_isolation_level(0)
                    cur.execute(sql)
                    conn.commit()
                    conn.set_isolation_level(1)
                else:
                    cur.execute(sql)
                    conn.commit()

            with conncur(dbname='heatmap_test',user='postgres', host='127.0.0.1', port=5432) as (conn, cur):
                sql = sql_blocks[4]
                cur.execute(sql)
                conn.commit()

    with conncur(dbname='heatmap_test',user='heatmapadmin', host='127.0.0.1', port=5432) as (conn, cur):
        for sql in sql_blocks[5:8]:
            print(sql)
            cur.execute(sql)
            conn.commit()

def main():
    setup_db()

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.13567708333333334, 'fraction_numerical': 0.019791666666666666, 'mean_word_length': 2.082664526484751, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 5, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3332128', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1165, 'n_words': 302}