<jupyter_start><jupyter_text>Make training and test datasets<jupyter_code>import wave
import contextlib
def get_duration(wav_path):
    with contextlib.closing(wave.open(wav_path,'r')) as f:
        frames = f.getnframes()
        rate = f.getframerate()
        duration = frames / float(rate)
    return duration, frames, rate
import os
from tqdm import tqdm
import json
with open('config.json') as f:
    data = f.read()
config = json.loads(data)
wg_segment_length = config["data_config"]["segment_length"]
wg_sampling_rate = config["data_config"]["sampling_rate"]
data_tts_dir = '/data2/sungjaecho/data_tts'
db_list = ['KETTS', 'KETTS2', 'KSS', 'KAB', 'NC']
wav_dirs = [os.path.join(data_tts_dir, db, db, 'wav') for db in db_list]
wav_dirs
voice_type_set = set()
for wav_dir in wav_dirs:
    wavs = sorted(os.listdir(wav_dir))
    for wav in wavs:
        splits = wav.split('_')
        if len(splits) == 3:
            voice_type = '{}_{}'.format(splits[0], splits[1])
        else:
            voice_type = splits[0]
        voice_type_set.add(voice_type)
print('voice_type_set', voice_type_set)
print('len(voice_type_set)', len(voice_type_set))
n_test_wav_per_voice_type = 10
max_duration = 10

cnt_wrong_sr = 0
cnt_short_wav = 0

test_wavs = dict()
train_wavs = dict()
for voice_type in voice_type_set:
    test_wavs[voice_type] = list()
for voice_type in voice_type_set:
    train_wavs[voice_type] = list()
    
for wav_dir in wav_dirs:
    wavs = sorted(os.listdir(wav_dir))
    for wav in tqdm(wavs):
        wav_path = os.path.join(wav_dir, wav)
        '''if get_duration(wav_path)[0] > max_duration:
            continue'''
        duration, n_frames, sr = get_duration(wav_path)
        if wg_sampling_rate != sr:
            cnt_wrong_sr += 1
            continue
        if n_frames < wg_segment_length:
            cnt_short_wav += 1
            continue
        
        splits = wav.split('_')
        if len(splits) == 3:
            voice_type = '{}_{}'.format(splits[0], splits[1])
        else:
            voice_type = splits[0]
        
        if len(test_wavs[voice_type]) < n_test_wav_per_voice_type:
            test_wavs[voice_type].append(wav_path)
        else:
            train_wavs[voice_type].append(wav_path)

for voice_type in voice_type_set:
    print('len(test_wavs[{}])'.format(voice_type), len(test_wavs[voice_type]))
for voice_type in voice_type_set:
     print('len(train_wavs[{}])'.format(voice_type), len(train_wavs[voice_type]))
        
print("Exclusion")
print("#samples with wrong sampling rate", cnt_wrong_sr)
print("#samples of short length", cnt_short_wav)
test_wav_paths = list()
for voice_type in voice_type_set:
    test_wav_paths += test_wavs[voice_type]

train_wav_paths = list()
for voice_type in voice_type_set:
    train_wav_paths += train_wavs[voice_type]
    
print('len(test_wav_list)', len(test_wav_paths))
print('len(train_wav_list)', len(train_wav_paths))
test_wav_paths
with open('test_files.txt', 'w') as f:
    w_string = ''
    for test_wav_path in test_wav_paths:
        w_string += test_wav_path
        w_string += '\n'
    w_string = w_string[:-1]
    f.write(w_string)
    
with open('train_files.txt', 'w') as f:
    w_string = ''
    for train_wav_path in train_wav_paths:
        w_string += train_wav_path
        w_string += '\n'
    w_string = w_string[:-1]
    f.write(w_string)
    
from shutil import copyfile

copyfile('test_files.txt', 'file_lists/01_KETTS_KETTS2_KSS_NC_KAD/test_files.txt')
copyfile('train_files.txt', 'file_lists/01_KETTS_KETTS2_KSS_NC_KAD/train_files.txt')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12208008898776418, 'fraction_numerical': 0.007508342602892102, 'mean_word_length': 3.5938697318007664, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '191457', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1399, 'n_tokens_neox': 1380, 'n_words': 252, 'path': 'dev_ipynb/01_make_train_test_sets.ipynb', 'repo_name': 'sungjae-cho/waveglow', 'size': '3596'}