<commit_before>from __future__ import absolute_import, print_function
from collections import Iterable, Mapping, OrderedDict

import atexit
import datetime
import time
import os
import re
import warnings

import alembic.ddl
import sqlalchemy
import six

DATABASE_NAME = os.environ.get("SCRAPERWIKI_DATABASE_NAME",
                               "sqlite:///scraperwiki.sqlite")

DATABASE_TIMEOUT = float(os.environ.get("SCRAPERWIKI_DATABASE_TIMEOUT", 300))
SECONDS_BETWEEN_COMMIT = 2
unicode = type(u'')

class Blob(bytes):

    """
    Represents a blob as a string.
    """
PYTHON_SQLITE_TYPE_MAP = {
    six.text_type: sqlalchemy.types.Text,
    str: sqlalchemy.types.Text,
    int: sqlalchemy.types.BigInteger,
    bool: sqlalchemy.types.Boolean,
    float: sqlalchemy.types.Float,

    datetime.date: sqlalchemy.types.Date,
    datetime.datetime: sqlalchemy.types.DateTime,

    Blob: sqlalchemy.types.LargeBinary,
}

if bytes is not str:
    # On 2.7, bytes *is* str, so we don't want to overwrite that.
    PYTHON_SQLITE_TYPE_MAP[bytes] = sqlalchemy.types.LargeBinary

try:
    PYTHON_SQLITE_TYPE_MAP[long] = sqlalchemy.types.BigInteger
except NameError:
    pass


class _State(object):

    """
    This class maintains global state relating to the database such as
    connection. It does not form part of the public interface.
    """
    db_path = DATABASE_NAME
    engine = None
    _connection = None
    _transaction = None
    metadata = None
    table = None
    # Whether or not we need to create the table. It's set by
    # _set_table(); it's left unassigned here to catch
    # accidental uses of it.
    # table_pending = None
    vars_table_name = 'swvariables'
    last_commit = None
    echo = False

    @classmethod
    def connection(cls):
        if cls._connection is None:
            create = sqlalchemy.create_engine
            cls.engine = create(cls.db_path, echo=cls.echo,
                                connect_args={'timeout': DATABASE_TIMEOUT})
            cls._connection = cls.engine.connect()
            cls.new_transaction()
        if cls.table is None:
            cls.reflect_metadata()
            cls.table = sqlalchemy.Table('swdata', _State.metadata,
                                         extend_existing=True)
        if cls._transaction is None:
            cls.new_transaction()
        return cls._connection

    @classmethod
    def new_transaction(cls):
        cls.last_commit = time.time()
        if cls._transaction is not None:
            cls._transaction.commit()
        cls._transaction = cls._connection.begin()

    @classmethod
    def reflect_metadata(cls):
        if cls.metadata is None:
            cls.metadata = sqlalchemy.MetaData(bind=cls.engine)
        cls.metadata.reflect()

    @classmethod
    def check_last_committed(cls):
        if time.time() - cls.last_commit > SECONDS_BETWEEN_COMMIT:
            cls.new_transaction()


class Transaction(object):

    """
    This context manager must be used when other services need
    to connect to the database.
    """

    def __enter__(self):
        _State.connection()
        _State.new_transaction()

    def __exit__(self, *args):
        _State._transaction.commit()
        _State._transaction = None


@atexit.register
def commit_transactions():
    """
    Ensure any outstanding transactions are committed on exit
    """
    if _State._transaction is not None:
        _State._transaction.commit()
        _State._transaction = None


def execute(query, data=None):
    """
    Execute an arbitrary SQL query given by query, returning any
    results as a list of OrderedDicts. A list of values can be supplied as an,
    additional argument, which will be substituted into question marks in the
    query.
    """
    connection = _State.connection()
    _State.new_transaction()

    if data is None:
        data = []

    result = connection.execute(query, data)

    _State.table = None
    _State.metadata = None
    try:
        del _State.table_pending
    except AttributeError:
        pass

    if not result.returns_rows:
        return {u'data': [], u'keys': []}

    return {u'data': result.fetchall(), u'keys': list(result.keys())}<commit_msg>Added conditional check on 'atexit' event function: the Python interpreter may remove the '_State' class before running the 'atexit' event actions, which yields a stacktrace with a 'NoneType has no _transation attribute' error. Checking on the existence of the '_State' class, and then on its attributes ensures the NoneType error does not appear.<commit_after>from __future__ import absolute_import, print_function
from collections import Iterable, Mapping, OrderedDict

import atexit
import datetime
import time
import os
import re
import warnings

import alembic.ddl
import sqlalchemy
import six

DATABASE_NAME = os.environ.get("SCRAPERWIKI_DATABASE_NAME",
                               "sqlite:///scraperwiki.sqlite")

DATABASE_TIMEOUT = float(os.environ.get("SCRAPERWIKI_DATABASE_TIMEOUT", 300))
SECONDS_BETWEEN_COMMIT = 2
unicode = type(u'')

class Blob(bytes):

    """
    Represents a blob as a string.
    """
PYTHON_SQLITE_TYPE_MAP = {
    six.text_type: sqlalchemy.types.Text,
    str: sqlalchemy.types.Text,
    int: sqlalchemy.types.BigInteger,
    bool: sqlalchemy.types.Boolean,
    float: sqlalchemy.types.Float,

    datetime.date: sqlalchemy.types.Date,
    datetime.datetime: sqlalchemy.types.DateTime,

    Blob: sqlalchemy.types.LargeBinary,
}

if bytes is not str:
    # On 2.7, bytes *is* str, so we don't want to overwrite that.
    PYTHON_SQLITE_TYPE_MAP[bytes] = sqlalchemy.types.LargeBinary

try:
    PYTHON_SQLITE_TYPE_MAP[long] = sqlalchemy.types.BigInteger
except NameError:
    pass


class _State(object):

    """
    This class maintains global state relating to the database such as
    connection. It does not form part of the public interface.
    """
    db_path = DATABASE_NAME
    engine = None
    _connection = None
    _transaction = None
    metadata = None
    table = None
    # Whether or not we need to create the table. It's set by
    # _set_table(); it's left unassigned here to catch
    # accidental uses of it.
    # table_pending = None
    vars_table_name = 'swvariables'
    last_commit = None
    echo = False

    @classmethod
    def connection(cls):
        if cls._connection is None:
            create = sqlalchemy.create_engine
            cls.engine = create(cls.db_path, echo=cls.echo,
                                connect_args={'timeout': DATABASE_TIMEOUT})
            cls._connection = cls.engine.connect()
            cls.new_transaction()
        if cls.table is None:
            cls.reflect_metadata()
            cls.table = sqlalchemy.Table('swdata', _State.metadata,
                                         extend_existing=True)
        if cls._transaction is None:
            cls.new_transaction()
        return cls._connection

    @classmethod
    def new_transaction(cls):
        cls.last_commit = time.time()
        if cls._transaction is not None:
            cls._transaction.commit()
        cls._transaction = cls._connection.begin()

    @classmethod
    def reflect_metadata(cls):
        if cls.metadata is None:
            cls.metadata = sqlalchemy.MetaData(bind=cls.engine)
        cls.metadata.reflect()

    @classmethod
    def check_last_committed(cls):
        if time.time() - cls.last_commit > SECONDS_BETWEEN_COMMIT:
            cls.new_transaction()


class Transaction(object):

    """
    This context manager must be used when other services need
    to connect to the database.
    """

    def __enter__(self):
        _State.connection()
        _State.new_transaction()

    def __exit__(self, *args):
        _State._transaction.commit()
        _State._transaction = None


@atexit.register
def commit_transactions():
    """
    Ensure any outstanding transactions are committed on exit
    """
    if _State is not None and _State._transaction is not None:
        _State._transaction.commit()
        _State._transaction = None


def execute(query, data=None):
    """
    Execute an arbitrary SQL query given by query, returning any
    results as a list of OrderedDicts. A list of values can be supplied as an,
    additional argument, which will be substituted into question marks in the
    query.
    """
    connection = _State.connection()
    _State.new_transaction()

    if data is None:
        data = []

    result = connection.execute(query, data)

    _State.table = None
    _State.metadata = None
    try:
        del _State.table_pending
    except AttributeError:
        pass

    if not result.returns_rows:
        return {u'data': [], u'keys': []}

    return {u'data': result.fetchall(), u'keys': list(result.keys())}
{'fraction_non_alphanumeric': 0.0853755841787302, 'fraction_numerical': 0.0013678331243588283, 'mean_word_length': 2.674204355108878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3782405', 'n_tokens_mistral': 2611, 'n_tokens_neox': 2534, 'n_words': 730}