<jupyter_start><jupyter_text>Helper function to generate file list for Tacotron2<jupyter_code>CUT = 12 # 12s
SR = 16000 

def check_audio_length(path):
    audio, _ = sf.read(path)
    if len(audio) < (CUT * SR):
        return True
    else:
        return False
def random_sample(used_path, all_path):
    all_path.remove(used_path)
    path = random.sample(all_path, 1)[0]
    while True:
        if check_audio_length(path) == False:
            path = random.sample(all_path, 1)[0]
        else:
            return path<jupyter_output><empty_output><jupyter_text>Generate dataset dicts<jupyter_code>rootdir = '/data3/IEMOCAP/IEMOCAP_full_release'


def emo_script(wav_dir, trans_dir):
    trans_map = {}
    skipped_cnt = 0
    for sess_path in sorted(glob.glob(wav_dir)):
        sess = sess_path.split('/')[-1]
        trans_path = os.path.join(trans_dir, sess +'.txt')
        with open(trans_path, 'r') as file:
            while True:
                line = file.readline()
                if not line:
                    break
                if line[0] == 'S':
                    split_line = line.split(':')
                    transcript = split_line[-1][1:-2]
                    wav_file_name = split_line[0].split()[0]
                    wav_name = wav_file_name +'.wav'
                    try: # filter out wavname like Ses03F_impro06_MXX1
                        x = int(wav_file_name[-3:])
                    except:
                        continue
                    wav_path = str(os.path.join(sess_path, wav_name))
                    speaker_sess_id = wav_name.split('.')[0][:-3]
                    speaker_sess_path = str(os.path.join(sess_path, speaker_sess_id))
                    all_same_wav = glob.glob(speaker_sess_path + '*.wav')
                    sampled_path = random_sample(wav_path, all_same_wav)
                    y, _ = sf.read(wav_path)
                    y_alt, _ = sf.read(sampled_path)
                    if len(y) >= SR * CUT:
                        skipped_cnt += 1
                        continue
                    trans_map[wav_path] = [transcript, sampled_path]
    print('Number of Skipped files: ', skipped_cnt)
    return trans_map

def dict2list(test_dict, flag):
    output = []
    for key, value in test_dict.items():
        if flag == 0:
            s = key + '|' + value[0] + '\n'
        else:
            s = value[1] + '|' + 'nothing is here' + '\n'
        output.append(s)
    return output


# Test Code
# wav_dir = os.path.join(rootdir, 'Session4', 'sentences/wav/*')
# trans_dir = os.path.join(rootdir, 'Session4', 'dialog/transcriptions')
# trans_dict = emo_script(wav_dir, trans_dir)<jupyter_output><empty_output><jupyter_text>Generating .txt files for train, val, test<jupyter_code>session_list = ['Session1', 'Session2', 'Session3', 'Session4', 'Session5']

for session in session_list:
    wav_dir = os.path.join(rootdir, session, 'sentences/wav/*')
    trans_dir = os.path.join(rootdir, session, 'dialog/transcriptions')
    if session == 'Session4':
        val_dict = emo_script(wav_dir, trans_dir)
        val_output = dict2list(val_dict, 0)
        val_output_alt = dict2list(val_dict, 1)
        with open('filelists/IEMOCAP/val_filelist.txt', 'w') as file:
            file.writelines(val_output)
        with open('filelists/IEMOCAP/val_filelist_alt.txt', 'w') as file:
            file.writelines(val_output_alt)
    elif session == 'Session5':
        test_dict = emo_script(wav_dir, trans_dir)
        test_output = dict2list(test_dict, 0)
        test_output_alt = dict2list(test_dict, 1)
        with open('filelists/IEMOCAP/test_filelist.txt', 'w') as file:
            file.writelines(test_output)
        with open('filelists/IEMOCAP/test_filelist_alt.txt', 'w') as file:
            file.writelines(test_output_alt)
    else:
        train_dict = emo_script(wav_dir, trans_dir)
        train_output = dict2list(train_dict, 0)
        train_output_alt = dict2list(train_dict, 1)
        with open('filelists/IEMOCAP/train_filelist.txt', 'a') as file:
            file.writelines(train_output)
        with open('filelists/IEMOCAP/train_filelist_alt.txt', 'a') as file:
            file.writelines(train_output_alt)<jupyter_output>Number of Skipped files:  74
Number of Skipped files:  63
Number of Skipped files:  44
Number of Skipped files:  54
Number of Skipped files:  77<jupyter_text>Quality Check<jupyter_code>original_path = './filelists/IEMOCAP'
for ftype in ['train', 'val', 'test']:
    file_path = original_path + '/' + ftype + '_filelist.txt'
    alt_path = original_path + '/' + ftype + '_filelist_alt.txt'
    with open(file_path, 'r') as a, open(alt_path, 'r') as b:
        content_a = a.readlines()
        content_b = b.readlines()
        for line_a, line_b in zip(content_a, content_b):
            path_wav_a = line_a.split('|')[0]
            path_wav_b = line_b.split('|')[0]
            
            if path_wav_a.split('.')[0][-4] == path_wav_b.split('.')[0][-4] \
                and line_a.split('|')[0] != line_b.split('|')[0] and check_audio_length(path_wav_a) \
                    and check_audio_length(path_wav_b):
                continue
            else:
                print('==============================')
                print('something wrong')
                print('ftype', ftype)
                print(line_a.split('|')[0].split('.')[0][-4])
                print(line_b.split('|')[0].split('.')[0][-4])
                print(line_a.split('|')[0])
                print(line_b.split('|')[0])
                print('==============================')
print('Finish Checking')<jupyter_output>Finish Checking<jupyter_text>Checking the number<jupyter_code>num_lines_train = sum(1 for line in open('filelists/IEMOCAP/train_filelist.txt'))
num_lines_val = sum(1 for line in open('filelists/IEMOCAP/val_filelist.txt'))
num_lines_test = sum(1 for line in open('filelists/IEMOCAP/test_filelist.txt'))
num_lines_train_alt = sum(1 for line in open('filelists/IEMOCAP/train_filelist_alt.txt'))
num_lines_val_alt = sum(1 for line in open('filelists/IEMOCAP/val_filelist_alt.txt'))
num_lines_test_alt = sum(1 for line in open('filelists/IEMOCAP/test_filelist_alt.txt'))

print('Check the numbers!')
print(num_lines_train)
print(num_lines_val)
print(num_lines_test)
print('====================')
print(num_lines_train_alt)
print(num_lines_val_alt)
print(num_lines_test_alt)
# Train count:  5788
# Valid count:  2103
# Test count:  2194
# total count:  10085<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13711656441717793, 'fraction_numerical': 0.01625766871165644, 'mean_word_length': 2.6147450110864745, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '557133', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2294, 'n_tokens_neox': 2219, 'n_words': 411, 'path': 'IEMOCAP_Preprocess.ipynb', 'repo_name': 'xin71/tacotron2', 'size': '6520'}