<jupyter_start><jupyter_text>Generate dataset<jupyter_code>from csrc.build import generate_from_audio
from csrc.configurations import DatasetConfig as DC
from utils import check_type, extract_audio, count_class
import decoder<jupyter_output><empty_output><jupyter_text>user configurations<jupyter_code># Kowning: the name of the audio is restrained to (name-lang).xxx, you can define the name as you like but remember to add the  standard language type as lang.
# You can name your subtitle file as your want."
test_clips = ("./src/src-test/test-eng.mp4", "./src/src-test/test.ass")

dbc = ("./src/src-dallas-buyers-club/dallas-buyers-club-eng.dts", "./src/src-dallas-buyers-club/dallas-buyers-club-eng.srt")
ab = ("./src/src-american-beauty/american-beauty-eng.ac3", "./src/src-american-beauty/american-beauty-eng.ass")
lstsb = ("./src/src-lstsb/lstsb-eng.ac3", "./src/src-lstsb/lstsb-eng.ass")
mi = ("./src/src-mission-impossible-iv/mission-impossible-iv.ac3", "./src/src-mission-impossible-iv/mission-impossible-iv.ass")
tdkr = ("./src/src-the-dark-knight-rises/the-dark-knight-rises-eng.aac", "./src/src-the-dark-knight-rises/the-dark-knight-rises-eng.ass")
tkoh = ("./src/src-the-kingdom-of-heaven/the-kingdom-of-heaven-eng.ac3", "./src/src-the-kingdom-of-heaven/the-kingdom-of-heaven-eng.ass")
tks = ("./src/src-the-kings-speech/the-kings-speech-eng.ac3", "./src/src-the-kings-speech/the-kings-speech-eng.ass")

dataset_source_long = [dbc, ab, lstsb, mi, tdkr, tkoh, tks]

dataset_source_short = [dbc, ab, lstsb]

dataset_source_medium = [dbc, ab, lstsb, mi, tdkr]

sources = [
    # (audio file path under src, subtitle file path correspondingly under src)\n",
    *dataset_source_long
]

# Destination folder for you to store current sources clips.
# It"s ok if you want to store clips directly under the data folder.
# and this won"t effect whether you want to separate the train/test folder by yourself.

dest_path = ("./data/standard-p3/")
_, _, _ = count_class(dest_path)<jupyter_output>Label 1 instances: 10964
Label 0 instances: 8070
Total clips: 19034<jupyter_text>Start auto generation<jupyter_code>for source_file_path, subtitle_file_path in sources: 
    if subtitle_file_path.split(".")[-1]=="ass":
        subtitle_decoder = decoder.ASSDecoder
    if subtitle_file_path.split(".")[-1]=="srt":
        subtitle_decoder = decoder.SRTDecoder   
    is_video = check_type(source_file_path)
    audio_file_path = extract_audio(source_file_path, format=DC.dataset_audio_format) if is_video else source_file_path
    generate_from_audio(audio_path=audio_file_path, sub_path=subtitle_file_path, dest_path=dest_path, sub_decoder=subtitle_decoder,  verbose=True)
_, _, _ = count_class(dest_path)<jupyter_output>Label 1 instances: 10964
Label 0 instances: 8070
Total clips: 19034
{'fraction_non_alphanumeric': 0.1373744619799139, 'fraction_numerical': 0.014705882352941176, 'mean_word_length': 4.8715789473684215, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '277874', 'license': 'MIT', 'n_tokens_mistral': 1066, 'n_tokens_neox': 1028, 'n_words': 200, 'path': 'build-dataset.ipynb', 'repo_name': 'EMUNES/ASFG-train', 'size': '2788'}