<jupyter_start><jupyter_text>Import Dependencies Replace github credentials before running<jupyter_code>!pip install pydub
# Mount google drive
from google.colab import drive
drive.mount('/content/gdrive')
os.chdir('/content/gdrive/MyDrive')
print("Working dir:", os.getcwd())
# Clone Speech project
!git clone https://github.com/wwongwk/speech-recognition-bias.git speech_rec_bias
!cp -R /content/speech_rec_bias /content/gdrive/My\ Drive/speech_rec_bias
!rm -R /content/speech_rec_bias<jupyter_output><empty_output><jupyter_text>Prepare Env<jupyter_code>import os
import sys
# Make sure you are running in the project folder
ROOT_FOLDER = os.getcwd() + "/web/"
DATA_SET_FOLDER = ROOT_FOLDER + "/test/"
RESULTS_FOLDER = ROOT_FOLDER + '/results/'
os.chdir(ROOT_FOLDER)
sys.path.append(os.path.abspath(ROOT_FOLDER))
print("Working dir: " + ROOT_FOLDER)

wav_seq = []
for root, dirs, files in os.walk(DATA_SET_FOLDER):
    for filename in files:
        wav_seq.append(filename)
print("Reading wav file number:", len(wav_seq))

import audio_handler as ah
import commercial_api as ca<jupyter_output>Working dir: /home/ubuntu/study/IS5452/final_project/web/
Reading wav file number: 745
Reading sound file...
/home/ubuntu/study/IS5452/final_project/web
Loading library...
Analyzed by: OpenVokaturi version 3.3 for open-source projects, 2019-05-30
Distributed under the GNU General Public License, version 3 or later
Initialized... lib: /home/ubuntu/study/IS5452/final_project/web/lib/lib_arm64.so
{'empath': 'https://api.webempath.net/v2/analyzeWav', 'deepaffect': 'https://proxy.api.deepaffects.com/audio/generic/api/v2/sync/recognise_emotion'}
{'empath': 'eCZq7YPVGsLnjS5MQp-NNJBTugv90bWVm_ntBYu0XaA', 'deepaffect': 'QUV3GNwPB7EPk5TWJQUjospamvusFGK3'}<jupyter_text>Run<jupyter_code>check_point = 0
for i in range(check_point, len(wav_seq)):
    wav = ah.readWavFromPath(DATA_SET_FOLDER + wav_seq[i])
    index = wav_seq[i].split('.')[0] # Data index
    audio_format = 'wav' # ogg also supported
    print("Current index : ",i, " Name: ", index)
    result = ca.emotion_detect(wav, index, audio_format)
    with open(RESULTS_FOLDER + index + '.json', 'w') as f:
        f.write(result)
        f.close()
    print('\n')
## Replace empath
import json
p = 640
for i in range(p, len(wav_seq)):
    wav = ah.readWavFromPath(DATA_SET_FOLDER + wav_seq[i])
    index = wav_seq[i].split('.')[0]
    audio_format = 'wav'
    print("Current index : ",i, " Name: ", index)
    result = ca.emotion_detect(wav, index, audio_format)
    print(result)
    j = None
    with open(RESULTS_FOLDER + index + '.json', 'r') as f:
        j = json.loads(f.read())
        print(j)
        f.close()
        
    with open(RESULTS_FOLDER + index + '.json', 'w') as f:
        j['empath'] = json.loads(result)['empath']
        f.write(json.dumps(j))
        f.close()
    print('\n')
# Do check
import json
count = 0
for root, dirs, files in os.walk(RESULTS_FOLDER):
    for filename in files:
        with open(RESULTS_FOLDER + filename, 'r') as f:
            try:
                content = f.read()
                j = json.loads(content)
                if len(content) == 0:
                    print(filename)
                if 'Happy' not in j['vokaturi']:
                    print("bad: ", filename)
                    count += 1
            except:
                pass
print("Checking done, bad: ", count)<jupyter_output>bad:  1090_WSI_SAD_XX.json
bad:  1075_WSI_SAD_XX.json
bad:  1046_ITS_SAD_XX.json
bad:  1007_TSI_FEA_XX.json
bad:  1037_IEO_FEA_LO.json
bad:  1088_DFA_SAD_XX.json
bad:  1088_ITS_DIS_XX.json
bad:  1065_ITS_SAD_XX.json
bad:  1088_IEO_SAD_MD.json
bad:  1034_IEO_FEA_LO.json
bad:  1025_ITH_DIS_XX.json
bad:  1027_IEO_HAP_LO.json
bad:  1054_TSI_SAD_XX.json
bad:  1081_TSI_SAD_XX.json
bad:  1075_IWW_DIS_XX.json
bad:  1033_IEO_SAD_MD.json
bad:  1014_ITH_SAD_XX.json
bad:  1087_ITS_FEA_XX.json
bad:  1004_ITH_SAD_XX.json
bad:  1090_ITS_SAD_XX.json
bad:  1034_TSI_ANG_XX.json
bad:  1027_ITH_SAD_XX.json
bad:  1040_WSI_SAD_XX.json
bad:  1025_IWL_SAD_XX.json
bad:  1007_ITH_FEA_XX.json
bad:  1091_TSI_DIS_XX.json
bad:  1077_IEO_SAD_HI.json
bad:  1015_ITH_DIS_XX.json
bad:  1085_ITS_SAD_XX.json
bad:  1025_TIE_SAD_XX.json
bad:  1075_TSI_DIS_XX.json
bad:  1053_IEO_SAD_MD.json
bad:  1075_IEO_SAD_LO.json
bad:  1054_IWL_SAD_XX.json
bad:  1032_IEO_SAD_LO.json
bad:  1040_IEO_SAD_MD.json
bad:  1087_ITH_DIS_XX.json
b[...]
{'fraction_non_alphanumeric': 0.12075215224286362, 'fraction_numerical': 0.044404168554599006, 'mean_word_length': 3.7885032537960956, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '542933', 'license': 'Apache-2.0', 'n_tokens_mistral': 1940, 'n_tokens_neox': 1793, 'n_words': 342, 'path': 'web/commercial_emotion_detection.ipynb', 'repo_name': 'wwongwk/speech-recognition-bias', 'size': '4414'}