<commit_before>from datetime import datetime, timedelta
from inspect import isgenerator
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict  # Python < 2.7 drop-in

from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean, types, Table, event


def guess_type(sample):
    if isinstance(sample, bool):
        return Boolean
    elif isinstance(sample, int):
        return Integer
    elif isinstance(sample, float):
        return Float
    elif isinstance(sample, datetime):
        return DateTime
    return UnicodeText


class ResultIter(object):
    """ SQLAlchemy ResultProxies are not iterable to get a
    list of dictionaries. This is to wrap them. """

    def __init__(self, result_proxies):
        if not isgenerator(result_proxies):
            result_proxies = iter((result_proxies, ))
        self.result_proxies = result_proxies

        self.count = 0
        if not self._next_rp():
            raise StopIteration

    def _next_rp(self):
        try:
            self.rp = next(self.result_proxies)
            self.count += self.rp.rowcount
            self.keys = list(self.rp.keys())
            return True
        except StopIteration:
            return False

    def __next__(self):
        row = self.rp.fetchone()
        if row is None:
            if self._next_rp():
                return next(self)
            else:
                # stop here
                raise StopIteration
        return OrderedDict(zip(self.keys, row))

    next = __next__

    def __iter__(self):
        return self


def sqlite_datetime_fix():
    class SQLiteDateTimeType(types.TypeDecorator):
        impl = types.Integer
        epoch = datetime(1970, 1, 1, 0, 0, 0)

        def process_bind_param(self, value, dialect):
            return (value / 1000 - self.epoch).total_seconds()

        def process_result_value(self, value, dialect):
            return self.epoch + timedelta(seconds=value / 1000)<commit_msg>Don't convert to datetime, if it is already datetime<commit_after>from datetime import datetime, timedelta
from inspect import isgenerator
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict  # Python < 2.7 drop-in

from sqlalchemy import Integer, UnicodeText, Float, DateTime, Boolean, types, Table, event


def guess_type(sample):
    if isinstance(sample, bool):
        return Boolean
    elif isinstance(sample, int):
        return Integer
    elif isinstance(sample, float):
        return Float
    elif isinstance(sample, datetime):
        return DateTime
    return UnicodeText


class ResultIter(object):
    """ SQLAlchemy ResultProxies are not iterable to get a
    list of dictionaries. This is to wrap them. """

    def __init__(self, result_proxies):
        if not isgenerator(result_proxies):
            result_proxies = iter((result_proxies, ))
        self.result_proxies = result_proxies

        self.count = 0
        if not self._next_rp():
            raise StopIteration

    def _next_rp(self):
        try:
            self.rp = next(self.result_proxies)
            self.count += self.rp.rowcount
            self.keys = list(self.rp.keys())
            return True
        except StopIteration:
            return False

    def __next__(self):
        row = self.rp.fetchone()
        if row is None:
            if self._next_rp():
                return next(self)
            else:
                # stop here
                raise StopIteration
        return OrderedDict(zip(self.keys, row))

    next = __next__

    def __iter__(self):
        return self


def sqlite_datetime_fix():
    class SQLiteDateTimeType(types.TypeDecorator):
        impl = types.Integer
        epoch = datetime(1970, 1, 1, 0, 0, 0)

        def process_bind_param(self, value, dialect):
            if isinstance(value, datetime):
                return value
            return (value / 1000 - self.epoch).total_seconds()

        def process_result_value(self, value, dialect):
            return self.epoch + timedelta(seconds=value / 1000)
{'fraction_non_alphanumeric': 0.07864077669902912, 'fraction_numerical': 0.009708737864077669, 'mean_word_length': 2.2706349206349206, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7191427', 'n_tokens_mistral': 1184, 'n_tokens_neox': 1106, 'n_words': 338}