<commit_before>from sqlalchemy import *
from migrate import *
import hashlib


def generate_hash(url_string):
    m = hashlib.sha256()
    m.update(url_string)
    return m.hexdigest()[:14]


def upgrade(migrate_engine):
    """We've changed the algo for hashing and need to update all hash_id

    The hash function is copied to this migration in case we need to change it
    later on. We need to loop through each url in url_hash, generate a new
    hash, and then update the matching hashes in:

        url_hash
        bmarks
        readable
        -- if sqlite -- readable_fts
    """
    meta = MetaData(migrate_engine)

    # get the list of all urls from url_hash table
    url_hash = Table('url_hash', meta, autoload=True)
    bmarks = Table('bmarks', meta, autoload=True)
    readable = Table('readable', meta, autoload=True)

    if 'sqlite' in migrate_engine.dialect.driver.lower():
        readable_fts = Table('readable', meta, autoload=True)
    else:
        readable_fts = None

    hashes = migrate_engine.execute(select([url_hash]))

    for orig in hashes:
        orig_hash = orig['hash_id']
        new_hash = generate_hash(orig['url'])

        up_bmarks = bmarks.update().where(bmarks.c.hash_id==orig_hash).values(hash_id=new_hash)
        up_url_hash = url_hash.update().where(url_hash.c.hash_id==orig_hash).values(hash_id=new_hash)
        up_readable = readable.update().where(readable.c.hash_id==orig_hash).values(hash_id=new_hash)

        migrate_engine.execute(up_bmarks)
        migrate_engine.execute(up_url_hash)
        migrate_engine.execute(up_readable)

        if readable_fts is not None:
            migrate_engine.execute(<commit_msg>Fix bug where the readable_fts table was the wrong<commit_after>from sqlalchemy import *
from migrate import *
import hashlib


def generate_hash(url_string):
    m = hashlib.sha256()
    m.update(url_string)
    return m.hexdigest()[:14]


def upgrade(migrate_engine):
    """We've changed the algo for hashing and need to update all hash_id

    The hash function is copied to this migration in case we need to change it
    later on. We need to loop through each url in url_hash, generate a new
    hash, and then update the matching hashes in:

        url_hash
        bmarks
        readable
        -- if sqlite -- readable_fts
    """
    meta = MetaData(migrate_engine)

    # get the list of all urls from url_hash table
    url_hash = Table('url_hash', meta, autoload=True)
    bmarks = Table('bmarks', meta, autoload=True)
    readable = Table('readable', meta, autoload=True)

    if 'sqlite' in migrate_engine.dialect.driver.lower():
        readable_fts = Table('readable_fts', meta, autoload=True)
    else:
        readable_fts = None

    hashes = migrate_engine.execute(select([url_hash]))

    for orig in hashes:
        orig_hash = orig['hash_id']
        new_hash = generate_hash(orig['url'])

        up_bmarks = bmarks.update().where(bmarks.c.hash_id==orig_hash).values(hash_id=new_hash)
        up_url_hash = url_hash.update().where(url_hash.c.hash_id==orig_hash).values(hash_id=new_hash)
        up_readable = readable.update().where(readable.c.hash_id==orig_hash).values(hash_id=new_hash)

        migrate_engine.execute(up_bmarks)
        migrate_engine.execute(up_url_hash)
        migrate_engine.execute(up_readable)

        if readable_fts is not None:
            migrate_engine.execute(
{'fraction_non_alphanumeric': 0.09585416054101735, 'fraction_numerical': 0.002940311673037342, 'mean_word_length': 3.279245283018868, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6434120', 'n_tokens_mistral': 1132, 'n_tokens_neox': 1063, 'n_words': 269}