<commit_before>import logging
import time
import re
import boto3
from boto3.dynamodb.conditions import Key
from io import StringIO
import json
from os import environ
from os.path import join, dirname, realpath

LOG = logging.getLogger()
LOG.setLevel(logging.WARN)
KEY_PREFIX = "s/solutions"
COMMANDS_TABLE_NAME = environ.get("COMMANDS_TABLE_NAME", "testing-cmdchallenge-db-commands")
BUCKET_NAME = environ.get("BUCKET_NAME", "testing.cmdchallenge.com")
SHARD_INDEX = int(environ.get("SHARD_INDEX", "0"))
NUM_SHARDS = int(environ.get("SHARD", "10"))
dir_path = dirname(realpath(__file__))
static_path = join(dir_path, "../../static")


def slug_slice(slugs):
    """
    Given a list of slugs returns a subset based the shard index
    and the number of shards
    """
    last_slug_index = len(slugs) - 1
    start_index = int((last_slug_index * SHARD_INDEX) / NUM_SHARDS)
    if (SHARD_INDEX + 1) == NUM_SHARDS:
        end_index = len(slugs)
    else:
        end_index = int((last_slug_index * (SHARD_INDEX + 1)) / NUM_SHARDS)
    return slugs[start_index:end_index]


def handler(event, context):
    cmds = set()
    challenges = json.loads(open(join(dir_path, "ch/all-challenges.json")).read())

    if environ.get('LOCAL'):
        b = boto3.session.Session(profile_name='cmdchallenge', region_name='us-east-1')
        s3 = b.client('s3')
        slugs = challenges
    else:
        s3 = boto3.client('s3')
        slugs = slug_slice(challenges)

    table = b.resource('dynamodb').Table(COMMANDS_TABLE_NAME)

    for slug_name in slugs:
        resp = table.query(
            IndexName="challenge_slug-correct_length-index",
            KeyConditionExpression=Key("challenge_slug").eq(slug_name) & Key("correct_length").lt(20000000000),
            ScanIndexForward=True,
        )
        data = resp["Items"]
        while "LastEvaluatedKey" in resp:
            resp = table.query(
                ExclusiveStartKey=resp["LastEvaluatedKey"],<commit_msg>Fix lambda_cron<commit_after>import logging
import time
import re
import boto3
from boto3.dynamodb.conditions import Key
from io import StringIO
import json
from os import environ
from os.path import join, dirname, realpath

LOG = logging.getLogger()
LOG.setLevel(logging.WARN)
KEY_PREFIX = "s/solutions"
COMMANDS_TABLE_NAME = environ.get("COMMANDS_TABLE_NAME", "testing-cmdchallenge-db-commands")
BUCKET_NAME = environ.get("BUCKET_NAME", "testing.cmdchallenge.com")
SHARD_INDEX = int(environ.get("SHARD_INDEX", "0"))
NUM_SHARDS = int(environ.get("SHARD", "10"))
dir_path = dirname(realpath(__file__))
static_path = join(dir_path, "../../static")


def slug_slice(slugs):
    """
    Given a list of slugs returns a subset based the shard index
    and the number of shards
    """
    last_slug_index = len(slugs) - 1
    start_index = int((last_slug_index * SHARD_INDEX) / NUM_SHARDS)
    if (SHARD_INDEX + 1) == NUM_SHARDS:
        end_index = len(slugs)
    else:
        end_index = int((last_slug_index * (SHARD_INDEX + 1)) / NUM_SHARDS)
    return slugs[start_index:end_index]


def handler(event, context):
    cmds = set()
    challenges = json.loads(open(join(dir_path, "ch/all-challenges.json")).read())

    if environ.get('LOCAL'):
        b = boto3.session.Session(profile_name='cmdchallenge', region_name='us-east-1')
        s3 = b.client('s3')
        table = b.resource('dynamodb').Table(COMMANDS_TABLE_NAME)
        slugs = challenges
    else:
        s3 = boto3.client('s3')
        table = boto3.resource('dynamodb').Table(COMMANDS_TABLE_NAME)
        slugs = slug_slice(challenges)


    for slug_name in slugs:
        resp = table.query(
            IndexName="challenge_slug-correct_length-index",
            KeyConditionExpression=Key("challenge_slug").eq(slug_name) & Key("correct_length").lt(20000000000),
            ScanIndexForward=True,
        )
        data = resp["Items"]
        while "LastEvaluatedKey" in resp:
            resp = table.query(
                ExclusiveStartKey=resp["LastEvaluatedKey"],
{'fraction_non_alphanumeric': 0.1229426433915212, 'fraction_numerical': 0.013216957605985038, 'mean_word_length': 3.5424688561721402, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3936975', 'n_tokens_mistral': 1383, 'n_tokens_neox': 1340, 'n_words': 268}