<commit_before>import uuid
from datetime import datetime
from hashlib import sha1

from datawire.core import db


class ModelCore(object):
    id = db.Column(db.Integer, primary_key=True)

    created_at = db.Column(db.DateTime, default=datetime.utcnow)
    updated_at = db.Column(db.DateTime, default=datetime.utcnow,
                           onupdate=datetime.utcnow)

    @classmethod
    def by_id(cls, id):
        q = db.session.query(cls).filter_by(id=id)
        return q.first()

    @classmethod
    def all(cls):
        return db.session.query(cls)


def make_token():
    return uuid.uuid4().get_hex()[15:]


def data_hash(data):
    # TODO: This has plenty of collisions, maybe get it right some time.
    if isinstance(data, (list, tuple)):
        return '||'.join(map(data_hash, data))
    if isinstance(data, datetime):
        return data.isoformat()
    if isinstance(data, dict):
        d = [(k, data_hash(v)) for k, v in sorted(data.items())]
        return sha1(data_hash(d)).hexdigest()
    return unicode(data)<commit_msg>Ignore unicode for hash. <commit_after>import uuid
from datetime import datetime
from hashlib import sha1

from datawire.core import db


class ModelCore(object):
    id = db.Column(db.Integer, primary_key=True)

    created_at = db.Column(db.DateTime, default=datetime.utcnow)
    updated_at = db.Column(db.DateTime, default=datetime.utcnow,
                           onupdate=datetime.utcnow)

    @classmethod
    def by_id(cls, id):
        q = db.session.query(cls).filter_by(id=id)
        return q.first()

    @classmethod
    def all(cls):
        return db.session.query(cls)


def make_token():
    return uuid.uuid4().get_hex()[15:]


def data_hash(data):
    # TODO: This has plenty of collisions, maybe get it right some time.
    if isinstance(data, (list, tuple)):
        return '||'.join(map(data_hash, data))
    if isinstance(data, datetime):
        return data.isoformat()
    if isinstance(data, dict):
        d = [(k, data_hash(v)) for k, v in sorted(data.items())]
        return sha1(data_hash(d)).hexdigest()
    return unicode(data).encode('ascii', 'ignore')
{'fraction_non_alphanumeric': 0.12394761459307764, 'fraction_numerical': 0.004677268475210477, 'mean_word_length': 3.0977011494252875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4189339', 'n_tokens_mistral': 694, 'n_tokens_neox': 670, 'n_words': 172}