<commit_before># -*- coding: utf-8 -*-
from core.utils import _tibanna_settings
from core.utils import TIBANNA_DEFAULT_STEP_FUNCTION_NAME
from core.utils import run_workflow
from core.pony_utils import Tibanna, FormatExtensionMap
from dcicutils.ff_utils import get_metadata


def handler(event, context):
    '''
    this is triggered on completed file upload from s3 and
    event will be set to file data.
    '''
    # get file name
    # print(event)

    input_json = make_input(event)
    print('EVENT: %s' % event)
    extra_file_format = get_extra_file_format(event)
    status = get_status(event)
    if extra_file_format:
        if status != 'to be uploaded by workflow':
            # for extra file-triggered md5 run, status check is skipped.
            input_json['input_files'][0]['format_if_extra'] = extra_file_format
            response = run_workflow(sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, input_json=input_json)
    else:
        # only run if status is uploading...
        if status == 'uploading' or event.get('force_run'):
            # trigger the step function to run
            response = run_workflow(sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, input_json=input_json)
        else:
            return {'info': 'status is not uploading'}

    # fix non json-serializable datetime startDate
    if response.get('startDate'):
        response['startDate'] = str(response['startDate'])
    print('RESPONSE (md5): %s' % response)
    return response


def get_extra_file_format(event):
    '''if the file extension matches the regular file format,
    returns None
    if it matches one of the format of an extra file,
    returns that format (e.g. 'pairs_px2'
    '''
    # guess env from bucket name
    bucket = event['Records'][0]['s3']['bucket']['name']<commit_msg>Drill down into response to get startDate<commit_after># -*- coding: utf-8 -*-
from core.utils import _tibanna_settings
from core.utils import TIBANNA_DEFAULT_STEP_FUNCTION_NAME
from core.utils import run_workflow
from core.pony_utils import Tibanna, FormatExtensionMap
from dcicutils.ff_utils import get_metadata


def handler(event, context):
    '''
    this is triggered on completed file upload from s3 and
    event will be set to file data.
    '''
    # get file name
    # print(event)

    input_json = make_input(event)
    print('EVENT: %s' % event)
    extra_file_format = get_extra_file_format(event)
    status = get_status(event)
    if extra_file_format:
        if status != 'to be uploaded by workflow':
            # for extra file-triggered md5 run, status check is skipped.
            input_json['input_files'][0]['format_if_extra'] = extra_file_format
            response = run_workflow(sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, input_json=input_json)
    else:
        # only run if status is uploading...
        if status == 'uploading' or event.get('force_run'):
            # trigger the step function to run
            response = run_workflow(sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, input_json=input_json)
        else:
            return {'info': 'status is not uploading'}

    # fix non json-serializable datetime startDate
    tibanna_resp = response.get('_tibanna', {}).get('response')
    if tibanna_resp and tibanna_resp.get('startDate'):
        tibanna_resp['startDate'] = str(tibanna_resp['startDate'])
    print('RESPONSE (md5): %s' % response)
    print('TIBANNA RESP: %s' % tibanna_resp)
    return response


def get_extra_file_format(event):
    '''if the file extension matches the regular file format,
    returns None
    if it matches one of the format of an extra file,
    returns that format (e.g. 'pairs_px2'
    '''
    # guess env from bucket name
    bucket = event['Records'][0]['s3']['bucket']['name']
{'fraction_non_alphanumeric': 0.09621535181236673, 'fraction_numerical': 0.0042643923240938165, 'mean_word_length': 3.3946135831381734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '658318', 'n_tokens_mistral': 1137, 'n_tokens_neox': 1119, 'n_words': 332}