<commit_before>import boto3
import ffmpy
import os

s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
dynamo = boto3.resource('dynamodb')
table = dynamo.Table('FT_SegmentState')
sqs = boto3.resource('sqs')
statusqueue = sqs.get_queue_by_name(QueueName='FT_status_queue')

def lambda_handler(event, context):
    # Get the object from the event and show its content type
    # This job is triggered by FT_VideoConversions

    Bucket = event[0]['dynamodb']['NewImage']['Bucket']
    Key = event[0]['dynamodb']['NewImage']['Key']
    ConversionID = event[0]['dynamodb']['NewImage']['ConversionID']
    QueueMessageID = event[0]['dynamodb']['NewImage']['QueueMessageID']

    print "Bucket/Key is {}{}".format(Bucket, Key)
    print "ConversionID is {}".format(ConversionID)

    if not VideoURL.endswith('/'):
        try:
            # Finagle S3 bucket naming conventions so that boto retrieves the correct file
            global split_key
            split_key = VideoURL.split('/')
            global file_name
            file_name = VideoURL[-1]
            global file_extension
            file_extension = os.path.splitext(file_name)[1]
            print "segmenting {} file".format(file_extension)

            # sqs.put_message(
            # QueueUrl=statusqueue
            # ReceiptHandle=StatusReceipt
            # status='Downloading'
            # )
            # Download the source file from s3
            s3_client.download_file(bucket, VideoURL, '/tmp/'+file_name)

<commit_msg>lambda handler updates<commit_after>import boto3
import ffmpy
import os

s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
dynamo = boto3.resource('dynamodb')
table = dynamo.Table('FT_SegmentState')
sqs = boto3.resource('sqs')
statusqueue = sqs.get_queue_by_name(QueueName='FT_status_queue')

def lambda_handler(event, context):
    # Get the object from the event and show its content type
    # This job is triggered by FT_VideoConversions

    Bucket = event[0]['dynamodb']['NewImage']['Bucket']
    Key = event[0]['dynamodb']['NewImage']['Key']
    ConversionID = event[0]['dynamodb']['NewImage']['ConversionID']
    QueueMessageID = event[0]['dynamodb']['NewImage']['QueueMessageID']

    print "Bucket/Key is {}{}".format(Bucket, Key)
    print "ConversionID is {}".format(ConversionID)

    if not Key.endswith('/'):
        try:
            # Finagle S3 bucket naming conventions so that boto retrieves the correct file
            global split_key
            split_key = Key.split('/')
            global file_name
            file_name = Key[-1]
            global file_extension
            file_extension = os.path.splitext(file_name)[1]
            print "segmenting {} file".format(file_extension)

            # sqs.put_message(
            # QueueUrl=statusqueue
            # ReceiptHandle=StatusReceipt
            # status='Downloading'
            # )
            # Download the source file from s3
            s3_client.download_file(bucket, Key, '/tmp/'+file_name)


{'fraction_non_alphanumeric': 0.12121212121212122, 'fraction_numerical': 0.011988011988011988, 'mean_word_length': 2.98937583001328, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '441886', 'n_tokens_mistral': 931, 'n_tokens_neox': 895, 'n_words': 200}