<commit_before>from time import time
from time import sleep
import argparse
import logging

from sqlalchemy.dialects import postgresql
from sqlalchemy import orm
from sqlalchemy import text
from sqlalchemy import sql

from app import db
from app import ti_queues
from app import failed_queue
from util import elapsed
from util import chunks
from util import safe_commit



def update_fn(cls, method_name, obj_id_list, shortcut_data=None, index=1):

    # we are in a fork!  dispose of our engine.
    # will get a new one automatically
    # if is pooling, need to do .dispose() instead
    db.engine.dispose()

    start = time()

    print u"obj_id_list: {}".format(obj_id_list)

    q = db.session.query(cls).options(orm.undefer('*')).filter(cls.id.in_(obj_id_list))

    obj_rows = q.all()

    num_obj_rows = len(obj_rows)
    print "{repr}.{method_name}() got {num_obj_rows} objects in {elapsed} seconds".format(
        repr=cls.__name__,
        method_name=method_name,
        num_obj_rows=num_obj_rows,
        elapsed=elapsed(start)
    )

    for count, obj in enumerate(obj_rows):
        start_time = time()

        if obj is None:
            return None

        method_to_run = getattr(obj, method_name)

        print u"\n***\n{count}: starting {repr}.{method_name}() method".format(
            count=count + (num_obj_rows*index),<commit_msg>don't print obj_ids for now<commit_after>from time import time
from time import sleep
import argparse
import logging

from sqlalchemy.dialects import postgresql
from sqlalchemy import orm
from sqlalchemy import text
from sqlalchemy import sql

from app import db
from app import ti_queues
from app import failed_queue
from util import elapsed
from util import chunks
from util import safe_commit



def update_fn(cls, method_name, obj_id_list, shortcut_data=None, index=1):

    # we are in a fork!  dispose of our engine.
    # will get a new one automatically
    # if is pooling, need to do .dispose() instead
    db.engine.dispose()

    start = time()

    # print u"obj_id_list: {}".format(obj_id_list)

    q = db.session.query(cls).options(orm.undefer('*')).filter(cls.id.in_(obj_id_list))

    obj_rows = q.all()

    num_obj_rows = len(obj_rows)
    print "{repr}.{method_name}() got {num_obj_rows} objects in {elapsed} seconds".format(
        repr=cls.__name__,
        method_name=method_name,
        num_obj_rows=num_obj_rows,
        elapsed=elapsed(start)
    )

    for count, obj in enumerate(obj_rows):
        start_time = time()

        if obj is None:
            return None

        method_to_run = getattr(obj, method_name)

        print u"\n***\n{count}: starting {repr}.{method_name}() method".format(
            count=count + (num_obj_rows*index),
{'fraction_non_alphanumeric': 0.09409190371991247, 'fraction_numerical': 0.0007293946024799417, 'mean_word_length': 3.3678343949044587, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7494602', 'n_tokens_mistral': 912, 'n_tokens_neox': 870, 'n_words': 259}