<commit_before>"""empty message

Revision ID: 0082_set_international
Revises: 0080_fix_rate_start_date
Create Date: 2017-05-05 15:26:34.621670

"""

# revision identifiers, used by Alembic.
from datetime import datetime

revision = '0082_set_international'
down_revision = '0080_fix_rate_start_date'

from alembic import op
import sqlalchemy as sa


def upgrade():
    conn = op.get_bind()
    start = datetime.utcnow()
    all_notifications = "select id from notification_history where international is null limit 10000"

    results = conn.execute(all_notifications)
    res = results.fetchall()

    conn.execute("update notifications set international = False where id in ({})".format(all_notifications))
    conn.execute("update notification_history set international = False where id in ({})".format(all_notifications))

    while len(res) > 0:
        conn.execute("update notifications set international = False where id in ({})".format(all_notifications))
        conn.execute("update notification_history set international = False where id in ({})".format(all_notifications))
        results = conn.execute(all_notifications)
        res = results.fetchall()
    end = datetime.utcnow()
    print("Started at: {}   ended at: {}".format(start, end))

def downgrade():
    # There is no way to downgrade this update.
    pass<commit_msg>Update the script to set the international flag to do the notifications and notification_history in separate loops. It takes about 1.5 minutes to update 27,000 notifications and 27,000 notification_history. The update is a row level lock so will only affect updates to the same row. This is unlikely as the data being updated should be older than 3 days. The second scripts updates the table to set international as not null, to make the model.<commit_after>"""empty message

Revision ID: 0082_set_international
Revises: 0081_noti_status_as_enum
Create Date: 2017-05-05 15:26:34.621670

"""
from datetime import datetime
from alembic import op

# revision identifiers, used by Alembic.
revision = '0082_set_international'
down_revision = '0081_noti_status_as_enum'


def upgrade():
    conn = op.get_bind()
    start = datetime.utcnow()
    notification_history = "select id from notification_history where international is null limit 10000"

    results = conn.execute(notification_history)
    res = results.fetchall()

    while len(res) > 0:
        conn.execute("update notification_history set international = False where id in ({})".format(
            notification_history))
        results = conn.execute(notification_history)
        res = results.fetchall()

    notifications = "select id from notifications where international is null limit 10000"
    results2 = conn.execute(notifications)
    res2 = results2.fetchall()
    while len(res2) > 0:
        conn.execute("update notifications set international = False where id in ({})".format(notifications))

        results2 = conn.execute(notifications)
        res2 = results2.fetchall()

    end = datetime.utcnow()
    print("Started at: {}   ended at: {}".format(start, end))


def downgrade():
    # There is no way to downgrade this update.
    pass
{'fraction_non_alphanumeric': 0.08723135271807839, 'fraction_numerical': 0.0347661188369153, 'mean_word_length': 4.12965964343598, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2290', 'n_tokens_mistral': 955, 'n_tokens_neox': 853, 'n_words': 304}