<commit_before># -*- coding: utf-8 -*-
#
# database.py
import os
import pg8000
import urlparse

## ------------------------------------------------- Database parts ----- ##
urlparse.uses_netloc.append("postgres")

class Database:
    def __init__(self):
        self._connect_db()

    def _connect_db(self):
        self.__url = urlparse.urlparse(os.environ["DATABASE_URL"])
        self.db = pg8000.connect(
                database=self.__url.path[1:],
                host=self.__url.hostname,
                port=self.__url.port,
                user=self.__url.username,
                password=self.__url.password,
                ssl=True
            )   

    def close_connection(self):
        if self.db is not None:
            self.db.close()
        self.db = None


    def query(self, q, args=None, commit=False):
        """Perform a query returning the database cursor if success else None.

        Use db_select for SELECT queries.
        Wrap the query with a try/except, catch the error, and return
        False if the query fails.
        """
        if self.db is None:
            self._connect_db()
        cur = self.db.cursor()
        try:
            cur.execute(q, args)
            if commit:
                self.db.commit()
        except pg8000.ProgrammingError as e:
            print "error:", e
            raise
        return cur


    def select(self, q, args=None, columns=None):
        """Return the result of a select query as an array of dictionaries.

        Each dictionary has keys taken from the 'columns' argument, or else
        'col0' ... 'colN-1' for the N columns returned.

        If there is an error with the query, return None.

        Keyword arguments
        args -- passed to pg8000.cursor.execute() for a secure
                parameterized query.
                We use the default format: SELECT * FROM TABLE WHERE col1 = '%s'
        """
        try:
            cur = self.query(q, args=args)
            if cur is None:
                return None
            else:
                results = cur.fetchall()
                cur.close()
        except pg8000.ProgrammingError as e:
            print "error:", e
            self.db.rollback()
            if cur is not None:
                cur.close()
            raise
        if results is None or len(results) == 0:
            return None
        elif len(results[0]) > len(columns):
            columns = list(columns) + [
                    "col%d" % i for i in range(len(columns),len(results))]
        elif len(results[0]) < len(columns):
            columns = columns[0:len(results[0])]

        return [dict(zip(columns, result)) for result in results]

<commit_msg>create cursor outside of if statement<commit_after># -*- coding: utf-8 -*-
#
# database.py
import os
import pg8000
import urlparse

## ------------------------------------------------- Database parts ----- ##
urlparse.uses_netloc.append("postgres")

class Database:
    def __init__(self):
        self._connect_db()

    def _connect_db(self):
        self.__url = urlparse.urlparse(os.environ["DATABASE_URL"])
        self.db = pg8000.connect(
                database=self.__url.path[1:],
                host=self.__url.hostname,
                port=self.__url.port,
                user=self.__url.username,
                password=<PASSWORD>,
                ssl=True
            )   

    def close_connection(self):
        if self.db is not None:
            self.db.close()
        self.db = None


    def query(self, q, args=None, commit=False):
        """Perform a query returning the database cursor if success else None.

        Use db_select for SELECT queries.
        Wrap the query with a try/except, catch the error, and return
        False if the query fails.
        """
        if self.db is None:
            self._connect_db()
        cur = self.db.cursor()
        try:
            cur.execute(q, args)
            if commit:
                self.db.commit()
        except pg8000.ProgrammingError as e:
            print "error:", e
            raise
        return cur


    def select(self, q, args=None, columns=None):
        """Return the result of a select query as an array of dictionaries.

        Each dictionary has keys taken from the 'columns' argument, or else
        'col0' ... 'colN-1' for the N columns returned.

        If there is an error with the query, return None.

        Keyword arguments
        args -- passed to pg8000.cursor.execute() for a secure
                parameterized query.
                We use the default format: SELECT * FROM TABLE WHERE col1 = '%s'
        """
        cur = None
        try:
            cur = self.query(q, args=args)
            if cur is None:
                return None
            else:
                results = cur.fetchall()
                cur.close()
        except pg8000.ProgrammingError as e:
            print "error:", e
            self.db.rollback()
            if cur is not None:
                cur.close()
            raise
        if results is None or len(results) == 0:
            return None
        elif len(results[0]) > len(columns):
            columns = list(columns) + [
                    "col%d" % i for i in range(len(columns),len(results))]
        elif len(results[0]) < len(columns):
            columns = columns[0:len(results[0])]

        return [dict(zip(columns, result)) for result in results]


{'fraction_non_alphanumeric': 0.11407053637031594, 'fraction_numerical': 0.011021307861866276, 'mean_word_length': 1.543204110228865, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1959925', 'n_tokens_mistral': 1511, 'n_tokens_neox': 1421, 'n_words': 463}