<commit_before># -*- encoding: utf-8

import os

import boto3
import botocore
from wellcome_aws_utils.dynamo_event import DynamoEvent
from wellcome_aws_utils.lambda_utils import log_on_error

from reindex_shard_config import create_reindex_shard


@log_on_error
def main(event, _ctxt=None, dynamodb_client=None):
    dynamodb_client = dynamodb_client or boto3.client('dynamodb')
    table_name = os.environ['TABLE_NAME']

    for record in event['Records']:
        dynamo_event = DynamoEvent(record)
        row = dynamo_event.new_image(deserialize_values=True)

        if not row:
            print("no NewImage key in dynamo update event, skipping")
            continue

        source_name = row['sourceName']
        source_id = row['id']
        new_reindex_shard = create_reindex_shard(
            source_name=row['sourceName'],
            source_id=source_id
        )

        # If the reindex shard doesn't match the current schema, we'll
        # create a new one.
        if row.get('reindexShard') == new_reindex_shard and 'reindexVersion' in row:
            print(<commit_msg>Add a TODO note<commit_after># -*- encoding: utf-8

import os

import boto3
import botocore
from wellcome_aws_utils.dynamo_event import DynamoEvent
from wellcome_aws_utils.lambda_utils import log_on_error

from reindex_shard_config import create_reindex_shard


@log_on_error
def main(event, _ctxt=None, dynamodb_client=None):
    dynamodb_client = dynamodb_client or boto3.client('dynamodb')

    # TODO: Isn't the table name passed in the record?
    table_name = os.environ['TABLE_NAME']

    for record in event['Records']:
        dynamo_event = DynamoEvent(record)
        row = dynamo_event.new_image(deserialize_values=True)

        if not row:
            print("no NewImage key in dynamo update event, skipping")
            continue

        source_name = row['sourceName']
        source_id = row['id']
        new_reindex_shard = create_reindex_shard(
            source_name=row['sourceName'],
            source_id=source_id
        )

        # If the reindex shard doesn't match the current schema, we'll
        # create a new one.
        if row.get('reindexShard') == new_reindex_shard and 'reindexVersion' in row:
            print(
{'fraction_non_alphanumeric': 0.0791111111111111, 'fraction_numerical': 0.0026666666666666666, 'mean_word_length': 3.1151736745886653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6439523', 'n_tokens_mistral': 717, 'n_tokens_neox': 695, 'n_words': 175}