<jupyter_start><jupyter_text>If you are using colab<jupyter_code># # link to google drive, click on the given link and choose the google drive account you would like to be available to you 

# from google.colab import drive
# drive.mount('/content/gdrive/')
# %cd ../content/gdrive/MyDrive/colab-ssh/LiSTra/notebooks/<jupyter_output><empty_output><jupyter_text>Dataset Statistics<jupyter_code># imports

import os
import subprocess

import numpy as np

import tqdm

# You may need to run this in your terminal to install sox
# ! sudo apt-get install sox
def get_audio_length(path):
    "Get audio length in second"
    output = subprocess.check_output(
        ['soxi -D \"%s\"' % path.strip()], shell=True)
    return float(output)

def get_averall_stats(path = "../dataset/train.en-ln.csv", 
                            audio_path = "../dataset/english/wav_verse/"):

    list_wav = []
    list_src_lang = []
    list_tgt_lang = []

    # Count number of words
    number_word_src = 0
    number_word_tgt = 0

    # To keep track of unique words 
    unique_word_src = set()
    unique_word_tgt = set()
        
    # open the and read file on in the first repository             
    with open(path, 'r') as f:  # Open file for read
        for textline in f:
            audio, transc, transl = textline.split('\t')
            
            list_wav.append(get_audio_length(audio_path+"/"+audio))
            list_src_lang.append(len(transc.split()))
            list_tgt_lang.append(len(transl.split()))

            unique_word_src.update(transc.split())
            unique_word_tgt.update(transl.split())
            
    
    print(f"We had a total of {len(list_wav)} audio files")
    print(f"We had a total of {len(list_src_lang)} English files")
    print(f"We had a total of {len(list_tgt_lang)} Lingala files")
    
    # Stats specific for the two language at use, you may need to change it according to your study 
    print("\n#######################################################\n")
    
    print(f"We have an average of {np.mean(list_wav)} audio length")
    print(f"We have an average of {np.mean(list_src_lang)} English text length")
    print(f"We have an average of {np.mean(list_tgt_lang)} Lingala text length")

    print("\n#######################################################\n")
    
    print(f"We have a total of {np.sum(list_wav)} audio length")
    print(f"We have  {len(unique_word_src)} English unique word")
    print(f"We have  {len(unique_word_tgt)} Lingala unique word")
    
    return list_wav, list_src_lang, list_src_lang
list_wav_test, list_src_lang_test, list_src_lang_test = get_averall_stats(
                path = "../../../processed/tf_data/test.en-ln.csv", \
                audio_path = "../../../external/LiSTra/dataset/english/wav_verse")
list_wav_train, list_src_lang_train, list_src_lang_train = get_averall_stats(
                path = "../../../processed/tf_data/train.en-ln.csv",\
                audio_path = "../../../external/LiSTra/dataset/english/wav_verse")
# avg_audio_valid, avg_len_en_valid, avg_len_ln_valid = get_averall_stats(path = "../dataset/valid.en-ln.csv",\
#                                                                 audio_path = "../dataset/wav_verse/")<jupyter_output><empty_output><jupyter_text>Total duration wav in hours<jupyter_code># Total hours in training data
sum(list_wav_test)/3600
# Total hours in training data
sum(list_wav_train)/3600<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14384548861343327, 'fraction_numerical': 0.0028826751225136927, 'mean_word_length': 2.9521640091116175, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '405722', 'license': 'MIT', 'n_tokens_mistral': 1135, 'n_tokens_neox': 1109, 'n_words': 270, 'path': 'notebooks/LiSTra stats.ipynb', 'repo_name': 'Kabongosalomon/LiSTra', 'size': '3469'}