<jupyter_start><jupyter_text>1. Upload the mp4 file into S3.  2. Run a Transcribe job on the mp4 file, specifying the number of speakers ideally.  3. Copy the Transcribe job results to this notebook to parse.<jupyter_code>!aws s3 cp s3://mansplaining/transcribe-wav.json .
import json
import boto3
import sagemaker
import os
import pandas as pd
def get_data(file_name):
    with open(file_name) as f:
        data = json.load(f)
    return data
    
data = get_data('transcribe-wav.json')['results']
def get_times(data):
    # must return a list of tuples like (start, duration)
    rt = []
    for segment in data['speaker_labels']['segments']:
        start_time = float(segment['start_time'])
        end_time = float(segment['end_time'])
        
        # get a 1/2 second snippet after each speaker starts
        delta = .5
                
        rt.append((start_time, delta))
                  
    return rt

times = get_times(data)<jupyter_output><empty_output><jupyter_text>4. Configure an Elastic Transcoder job<jupyter_code>client = boto3.client('elastictranscoder')
# pipeline = client.create_pipeline(
#     Name='pipeline',
#     InputBucket='mansplaining',
#     OutputBucket='mansplaining',

# run this to get the pipeline ID
pipeline
def get_inputs(times):
    inputs = []
    
    for start_time, duration in times:
        
        json_obj = { 'Key': 'audio/meeting_audio.wav',
                    'TimeSpan': {
                        'StartTime': str(start_time),
                        'Duration': str(duration)}
                    }
        inputs.append(json_obj)
                    
    return inputs 
        
inputs = get_inputs(times)
def get_outputs(times):
    rt = []
    for idx in range(len(times)):
        obj = {'Key' : 'micro_clip_{}.wav'.format(idx),
               
        # configure this by looking at the console based on the type of audio file you want 
         'PresetId':'1351620000001-300300'}

        rt.append(obj)
        
    return rt

outputs = get_outputs(times)
# need to loop through batches of 30 
iterator = round(len(inputs)/30)

for i in range(iterator):
    lb = i * 30
    up = lb + 30
    if up > len(inputs):
        up = len(inputs)
        
#     try:
    response = client.create_job(
        # get this from where you printed out pipeline above 
        PipelineId='1568399213667-jzukmi',
        Inputs=inputs[lb:up],
        Outputs = outputs[lb:up] )
    
#     break
#     except:
#         print ('failed on range', lb, up)<jupyter_output><empty_output><jupyter_text>5. Copy results from Transcoding job to Notebook instance<jupyter_code>!aws s3 sync s3://mansplaining Data<jupyter_output><empty_output><jupyter_text>6. Loop through data and hit SageMaker endpoint<jupyter_code>%%bash

for i in {1..291};do 
    echo $i;
    encoded_string=$(base64 ./Data/micro_clip_$i.wav);payload="{\"instances\": [{\"audio\": {\"b64\": \"$encoded_string\"}}]}";
    echo $payload >Input/input_$i.json;
    aws sagemaker-runtime invoke-endpoint  --endpoint-name 'gender-clasifier'  --body fileb://Input/input_$i.json --content-type "application/json" --region us-east-1 Output/output_$i.json;
    
done<jupyter_output>1
{
    "ContentType": "application/json",
    "InvokedProductionVariant": "variant-name-1"
}
2
{
    "ContentType": "application/json",
    "InvokedProductionVariant": "variant-name-1"
}
3
{
    "ContentType": "application/json",
    "InvokedProductionVariant": "variant-name-1"
}
4
{
    "ContentType": "application/json",
    "InvokedProductionVariant": "variant-name-1"
}
5
{
    "ContentType": "application/json",
    "InvokedProductionVariant": "variant-name-1"
}
6
{
    "ContentType": "application/json",
    "InvokedProductionVariant": "variant-name-1"
}
7
{
    "ContentType": "application/json",
    "InvokedProductionVariant": "variant-name-1"
}
8
{
    "ContentType": "application/json",
    "InvokedProductionVariant": "variant-name-1"
}
9
{
    "ContentType": "application/json",
    "InvokedProductionVariant": "variant-name-1"
}
10
{
    "ContentType": "application/json",
    "InvokedProductionVariant": "variant-name-1"
}
11
{
    "ContentType": "application/json",
    "InvokedPro[...]<jupyter_text>7. Consolidate Results and Generate Statistics<jupyter_code>json_objs = []
male_count = 0
female_count = 0

for f in os.listdir('Output'):
    file = 'Output/{}'.format(f)

    with open(file) as fp:
        data = json.load(fp)
        pred = data['predictions'][0]['label']
        if 'female' in pred:
            female_count += 1
        else:
            male_count += 1
percent_male = 100 * male_count / (male_count + female_count)
percent_female = 100 * female_count / (male_count + female_count)


print ('Percentage of time men were speaking was {}%, while for women it was {}%.'.format(percent_male, percent_female))<jupyter_output>Percentage of time men were speaking was 20.89041095890411%, while for women it was 79.10958904109589%.
{'fraction_non_alphanumeric': 0.1265899454875833, 'fraction_numerical': 0.02705431051887745, 'mean_word_length': 3.0048504446240907, 'pattern_counts': {'":': 24, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '276066', 'license': 'MIT', 'n_tokens_mistral': 1700, 'n_tokens_neox': 1590, 'n_words': 386, 'path': 'Gender Classifier Notebook.ipynb', 'repo_name': 'EmilyWebber/mansplaining', 'size': '4953'}