<commit_before>import os
import sys
import time
import ConfigParser
import signal
from optparse import OptionParser
import logging.config
import logging
import transaction
from sqlalchemy import text
from sqlalchemy.exc import OperationalError
import datetime
import sys


log = logging.getLogger(__name__)
# Can be overwrite by a config file
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
log.addHandler(ch)


loop = True


def sigterm_handler(signal_number, stack_frame):
    global loop
    loop = False
    log.info('Stopping the process by signal %i' % signal_number)


def sigterm_kill_handler(signal_number, stack_frame):
    global loop
    loop = False
    log.info('Killing the process by signal %i' % signal_number)
    sys.exit(0)


def _lock_task(connection, models):
    select_query = """
    SELECT idtask, unique_key
     FROM task
    WHERE status='%s'
      AND pid IS NULL
    LIMIT 10
    """ % (
        models.TASK_STATUS_WAITING
    )

    rows = connection.execute(select_query)
    for row in rows:
        idtask = row[0]
        unique_key = row[1]

        unique_key_extra_query = ''
        if unique_key:
            unique_key_extra_query = '''
          AND unique_key NOT IN (
            SELECT unique_key
              FROM task
             WHERE status = '%s'
               AND unique_key = '%s'
          )''' % (
                models.TASK_STATUS_IN_PROGRESS,
                unique_key)

        query = """
        UPDATE task
           SET pid = %i,
               status = '%s',
               lock_date = '%s'
        WHERE idtask = %i
          AND pid IS NULL<commit_msg>Don't get many unique keys in the same query<commit_after>import os
import sys
import time
import ConfigParser
import signal
from optparse import OptionParser
import logging.config
import logging
import transaction
from sqlalchemy import text
from sqlalchemy.exc import OperationalError
import datetime
import sys


log = logging.getLogger(__name__)
# Can be overwrite by a config file
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
log.addHandler(ch)


loop = True


def sigterm_handler(signal_number, stack_frame):
    global loop
    loop = False
    log.info('Stopping the process by signal %i' % signal_number)


def sigterm_kill_handler(signal_number, stack_frame):
    global loop
    loop = False
    log.info('Killing the process by signal %i' % signal_number)
    sys.exit(0)


def _lock_task(connection, models):
    select_query = """
    SELECT MIN(idtask) AS idtask, unique_key
     FROM task
    WHERE status='%s'
      AND pid IS NULL
 GROUP BY COALESCE(unique_key, CAST(idtask AS VARCHAR(255)))
    LIMIT 5
    """ % (
        models.TASK_STATUS_WAITING
    )

    rows = connection.execute(select_query)
    for row in rows:
        idtask = row[0]
        unique_key = row[1]

        unique_key_extra_query = ''
        if unique_key:
            unique_key_extra_query = '''
          AND unique_key NOT IN (
            SELECT unique_key
              FROM task
             WHERE status = '%s'
               AND unique_key = '%s'
          )''' % (
                models.TASK_STATUS_IN_PROGRESS,
                unique_key)

        query = """
        UPDATE task
           SET pid = %i,
               status = '%s',
               lock_date = '%s'
        WHERE idtask = %i
          AND pid IS NULL
{'fraction_non_alphanumeric': 0.07655786350148368, 'fraction_numerical': 0.0035608308605341245, 'mean_word_length': 2.2919921875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '315553', 'n_tokens_mistral': 1007, 'n_tokens_neox': 965, 'n_words': 297}