<jupyter_start><jupyter_text><jupyter_code>on_colab = True
try:
    from google.colab import drive
except:
    on_colab = False
    
print("Running on colab:", on_colab)

import os
import pathlib

if on_colab:
    drive.mount('/content/gdrive') #, force_remount=True 
    BASE_DIR = "/content/gdrive/My Drive/Colab Notebooks"
    unet_path = os.path.join(BASE_DIR, "DenoiseUNet")
else:
    BASE_DIR = os.path.join(os.getcwd())
    unet_path = BASE_DIR

print("Base dir:", BASE_DIR)
#setup directories
#Data
data_dir = os.path.join(unet_path, "data")
nstd_dir = os.path.join(data_dir, "NSDTSEA")
os.makedirs(nstd_dir, exist_ok=True)

#Git
repo = "https://github.com/jbgh2/speech-denoising-wavenet"

#Training/test data
data_file = "DS_10283_1942.zip"
data_url = f"http://datashare.is.ed.ac.uk/download/{data_file}" 
target_file = os.path.join(data_dir, data_file)


#Clone repo into dir or pull
if not os.path.exists(os.path.join(unet_path, ".git")):
    !git clone {repo} '{unet_path}'
else:
    !git -C '{unet_path}' pull #-C changes dir first
#Download test/train set
if not os.path.isfile(target_file):
    !wget -O '{target_file}' {data_url}
else:
    print(f"Main zip found at {target_file}, skipping download")
#Unzip main zip file
zips = list(pathlib.Path(nstd_dir).glob('*.zip'))
if len(zips) == 0:    
    !unzip '{target_file}' -d '{nstd_dir}'
else:
    print(f"Found zip files in {nstd_dir}, skipping unzip of {target_file}")
#Unzip all the sub-zip files
import pathlib
zips = pathlib.Path(nstd_dir).glob('*.zip')
for z in zips:
    dataset_name, _ = os.path.splitext(os.path.basename(z))
    dataset_path = os.path.join(nstd_dir, dataset_name)
    if not os.path.exists(dataset_path):
        print(f"Unzipping {z} to {dataset_path}")
        !unzip '{z}' -d '{dataset_path}'
    else:
        print(f"{dataset_path} exists, skipping unzip")
#Fix the misaligned sub-directories. Move files up one level
for p in os.listdir(nstd_dir):
    #print(p)
    full_path = os.path.join(nstd_dir, p)
    if os.path.isdir(full_path):
        num_files = len(os.listdir(full_path))
        print(f"{num_files} files in {p}")
        if num_files < 3:
            tgt_dir = os.path.join(nstd_dir, p)
            src_dir = os.path.join(tgt_dir, p)
            print(f"Move files from {src_dir} to {tgt_dir}")

            if os.path.isdir(src_dir) and os.path.isdir(tgt_dir):
                !mv "{src_dir}"/* '{tgt_dir}/'  
                pass<jupyter_output>11572 files in clean_trainset_wav
11572 files in noisy_trainset_wav
825 files in noisy_testset_wav
825 files in clean_testset_wav
11574 files in trainset_txt
825 files in testset_txt
{'fraction_non_alphanumeric': 0.13366149185914428, 'fraction_numerical': 0.013631200302915562, 'mean_word_length': 3.6761061946902656, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '452098', 'license': 'MIT', 'n_tokens_mistral': 1052, 'n_tokens_neox': 1015, 'n_words': 229, 'path': 'Setup_UNet_on_Colab.ipynb', 'repo_name': 'jbgh2/speech-denoising-wavenet', 'size': '2641'}