<commit_before>from json import dumps
from sql import connect, run_sql

CHECK_FACTS_TABLE_SQL = '''
SELECT COUNT(1) FROM information_schema.tables WHERE table_name = 'facts';
'''
GET_TOPICS_SQL = 'SELECT DISTINCT topic FROM facts'
GET_LAST_N_FACTS_SQL = '''
    SELECT id, ts, topic, content
    FROM (SELECT id, ROUND(EXTRACT(epoch FROM ts))::int AS ts, topic, content
          FROM facts WHERE topic LIKE %%s ORDER BY id DESC LIMIT %s) t
    ORDER BY id ASC
'''
GET_AFTER_ID_FACTS_SQL = '''
    SELECT id, ROUND(EXTRACT(epoch FROM ts))::int AS ts, topic, content
          FROM facts WHERE topic LIKE %s AND id > %s ORDER BY id ASC
'''
INSERT_FACT_SQL = '''
    INSERT INTO facts (topic, ts, content) VALUES (%s, now(), %s);
'''

class Factspace:
    def __init__(self, url):
        self.url = url
        conn = connect(self.url)
        result = run_sql(conn, CHECK_FACTS_TABLE_SQL, results=True)[0][0]
        if result != 1:
            raise MissingTableError('Facts table not present.')

    def list_topics(self):
        conn = connect(self.url)
        result = run_sql(conn, GET_TOPICS_SQL, results=True)
        return map(lambda x: x[0], result)

    def last_n(self, topic, number):
        conn = connect(self.url)
        topic = self._translate_wildcards(topic)
        result = run_sql(conn, GET_LAST_N_FACTS_SQL % (number,),
                         results=True, parameters=[topic])
        return map(self._format_fact, result)

    def after_id(self, topic, id):
        conn = connect(self.url)
        topic = self._translate_wildcards(topic)
        result = run_sql(conn, GET_AFTER_ID_FACTS_SQL,
                         results=True, parameters=[topic, id])
        return map(self._format_fact, result)

    def add_fact(self, topic, fact):
        conn = connect(self.url)
        run_sql(conn, INSERT_FACT_SQL, results=False,
                parameters=[topic, dumps(fact)])

    def _format_fact(self, returned_fact):
        fact = dict(returned_fact[3])
        fact['combo_id'] = returned_fact[0]<commit_msg>close connections<commit_after>from json import dumps
from sql import connect, run_sql

CHECK_FACTS_TABLE_SQL = '''
SELECT COUNT(1) FROM information_schema.tables WHERE table_name = 'facts';
'''
GET_TOPICS_SQL = 'SELECT DISTINCT topic FROM facts'
GET_LAST_N_FACTS_SQL = '''
    SELECT id, ts, topic, content
    FROM (SELECT id, ROUND(EXTRACT(epoch FROM ts))::int AS ts, topic, content
          FROM facts WHERE topic LIKE %%s ORDER BY id DESC LIMIT %s) t
    ORDER BY id ASC
'''
GET_AFTER_ID_FACTS_SQL = '''
    SELECT id, ROUND(EXTRACT(epoch FROM ts))::int AS ts, topic, content
          FROM facts WHERE topic LIKE %s AND id > %s ORDER BY id ASC
'''
INSERT_FACT_SQL = '''
    INSERT INTO facts (topic, ts, content) VALUES (%s, now(), %s);
'''

class Factspace:
    def __init__(self, url):
        self.url = url
        conn = connect(self.url)
        result = run_sql(conn, CHECK_FACTS_TABLE_SQL, results=True)[0][0]
        if result != 1:
            raise MissingTableError('Facts table not present.')
        conn.close()

    def list_topics(self):
        conn = connect(self.url)
        result = run_sql(conn, GET_TOPICS_SQL, results=True)
        ret_val = map(lambda x: x[0], result)
        conn.close()
        return ret_val

    def last_n(self, topic, number):
        conn = connect(self.url)
        topic = self._translate_wildcards(topic)
        result = run_sql(conn, GET_LAST_N_FACTS_SQL % (number,),
                         results=True, parameters=[topic])
        ret_val = map(self._format_fact, result)
        conn.close()
        return ret_val

    def after_id(self, topic, id):
        conn = connect(self.url)
        topic = self._translate_wildcards(topic)
        result = run_sql(conn, GET_AFTER_ID_FACTS_SQL,
                         results=True, parameters=[topic, id])
        ret_val = map(self._format_fact, result)
        conn.close()
        return ret_val

    def add_fact(self, topic, fact):
        conn = connect(self.url)
        run_sql(conn, INSERT_FACT_SQL, results=False,
                parameters=[topic, dumps(fact)])
        conn.close()
        return

    def _format_fact(self, returned_fact):
        fact = dict(returned_fact[3])
        fact['combo_id'] = returned_fact[0]
{'fraction_non_alphanumeric': 0.11043802423112768, 'fraction_numerical': 0.0032618825722274, 'mean_word_length': 2.7924028268551235, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3778930', 'n_tokens_mistral': 1415, 'n_tokens_neox': 1344, 'n_words': 364}