<jupyter_start><jupyter_text>Mobysound whale custom parsing- This is a series of datasets with whale/dolphin echolocation clicks. They are all available on Mobysound.     - WAV files for vocalization that contains labels for species and vocalization. - This notebook creates a JSON corresponding to each WAV file.- Dataset origin:    - https://www.mobysound.org/<jupyter_code>from avgn.utils.general import prepare_env
prepare_env()<jupyter_output>env: CUDA_VISIBLE_DEVICES=GPU<jupyter_text>Import relevant packages<jupyter_code>from joblib import Parallel, delayed
from tqdm.autonotebook import tqdm
import pandas as pd
pd.options.display.max_columns = None
import librosa
from datetime import datetime
import numpy as np
import re
import avgn
from avgn.custom_parsing.picidae_woodpecker import generate_json
from avgn.utils.paths import DATA_DIR
DATASET_ID = "MOBYSOUND_WHALES"<jupyter_output><empty_output><jupyter_text>Load data in original format<jupyter_code># create a unique datetime identifier for the files output by this notebook
DT_ID = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
DT_ID
subfolders = {
        'Melon headed whale':{
            'folder': 'MellonHeaded_MTSTCS',
            'species': 'Peponocephala electra',
            'split': '.',
            'click_types' : [
                            ['good', '*Good.txt']
                        ]
        },
        'Sperm whale':{
            'folder': 'Sperm whales_Bahamas(AUTEC)-Annotated/',
            'species': 'Peponocephala electra',
            'split': '_',
            'click_types' : [
                            ['maybe', '*maybe.txt'],
                            ['good', '*short_good.txt']
                        ]
        },
        'Pilot whale':{
            'folder': 'Pilot_whales_Bahamas(AUTEC)-Annotated-NUWC',
            'species': 'Peponocephala electra',
            'split': '_',
            'click_types' : [
                                ['good', '*good.txt'],
                                ['maybe', '*maybe1.txt']
                        ]
        },
        'Risos whale':{
            'folder': 'Rissos-SCORE-annot',
            'species': 'Peponocephala electra',
            'click_types' : [
                            ['good', '*GoodSingle.txt'],
                            ['poor', '*PoorSingle.txt'],
                        ]
        },
        'Pacific whitesided dolphin':{
            'folder': 'PacWhitesidedDolphin',
            'split': '.',
            'species': 'Peponocephala electra',
            'click_types' : [
                            ['type1h', '*type 1h.txt.txt']
                        ]
        },
        'Blainsville\'s beaked whale':{
            'folder': 'Mesoplodon_Bahamas(AUTEC)-Annotated',
            'split': '.',
            'species': 'Peponocephala electra',
            'click_types' : [
                            ['good', '*/*on_final.log'],
                            ['maybe', '*/*maybe_final.log'],
                        ]
        }
}
DSLOC = avgn.utils.paths.Path('/mnt/cube/Datasets/mobysound/')
DSLOC<jupyter_output><empty_output><jupyter_text>create JSON for each species<jupyter_code>from avgn.utils.audio import get_samplerate
from avgn.utils.json import NoIndent, NoIndentEncoder
from IPython.display import display
# for each species
for common_name, species_dict in tqdm(subfolders.items()):
    # get the folder
    ds_folder = DSLOC / species_dict['folder']
    
    # create a df of wavs in that folder
    wavs = list(ds_folder.glob('*.wav'))
    wav_df = pd.DataFrame(columns = ['stem', 'path', 'rate', 'duration'])
    for wav in wavs:
        try:
            sr = get_samplerate(wav.as_posix())
        except:
            print(wav)
            continue
        wav_duration = librosa.get_duration(filename=wav)
        wav_df.loc[len(wav_df)] = [wav.stem, wav, sr, wav_duration]
    
    # create a dataframe of clicks for those wavs
    clicks_df = []
    for click_type, click_type_glob in species_dict['click_types']:
        good_clicks = list(ds_folder.glob(click_type_glob))
        good_clicks = [i for i in good_clicks if i.stem[0] != '.']
        for click_label_loc in good_clicks:
            good_clicks_df = pd.read_csv(click_label_loc, delimiter='\t')
            good_clicks_df['stem'] = click_label_loc.stem
            good_clicks_df['click_value'] = click_type
            clicks_df.append(good_clicks_df)
    if len(clicks_df) == 0:
        continue
    clicks_df = pd.concat(clicks_df)
    clicks_df.columns = [re.sub(' +', ' ', i) for i in clicks_df.columns]
    clicks_df['stem'] = [row.stem[:-(len(row.click_value)+1)] for idx, row in clicks_df.iterrows()]
    #clicks_df['stem'] = [i.split(species_dict['split'])[0] for i in clicks_df.stem]
        
    display(clicks_df[:3])
        
    # for each wav
    for idx, wavrow in tqdm(wav_df.iterrows()):
        wavclicks = clicks_df[clicks_df.stem == wavrow.stem]
        if len(wavclicks) == 0:
            print(wavrow.stem)
            continue
        if type(wavclicks[" end time"].values[0]) is np.ndarray:
            stop_times = list([i[0] for i in wavclicks[" end time"].values])
        else:
            stop_times = list(wavclicks[" end time"].values)
            
        start_times = list(wavclicks["% start time"].values)
        
        print(start_times[:3], stop_times[:3])

        
        if np.isnan(start_times[0]):
            start_times = list(wavclicks["%start time"].values)
              
        if np.isnan(stop_times[0]):
            stop_times = [i[0] for i in wavclicks.index.values]        
        
        print(start_times[:3], stop_times[:3])
        
        # make json
        json_dict = {}
        json_dict["indvs"] = {
            "UNK": {
                "clicks": {
                    "start_times": NoIndent(start_times),
                    "end_times": NoIndent(stop_times),
                #"peak_freq": NoIndent(list(wavclicks["peak freq"].values)),
                #"center_freq": NoIndent(list(wavclicks["centr. freq"].values)),
                "click_value": NoIndent(list(wavclicks["click_value"].values)),
                }
                
            }
        }
        
        # add species
        json_dict["species"] = species_dict['species']
        json_dict["common_name"] = common_name
        json_dict["wav_loc"] = wavrow.path.as_posix()
        json_dict["stem"] = wavrow.stem
        # rate and length
        json_dict["samplerate_hz"] = wavrow.rate
        json_dict["length_s"] = wavrow.duration

        # dump json
        json_txt = json.dumps(json_dict, cls=NoIndentEncoder, indent=2)
        
        # json save location
        wav_stem = wavrow.stem
        json_out = (
            DATA_DIR
            / "processed"
            / DATASET_ID
            / DT_ID
            / "JSON"
            / (wav_stem + ".JSON")
        )
        
        # save json
        avgn.utils.paths.ensure_dir(json_out.as_posix())
        print(json_txt, file=open(json_out.as_posix(), "w"))
        #print(json_txt)

wavclicks<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1226042841037204, 'fraction_numerical': 0.002677564825253664, 'mean_word_length': 1.9781787662610155, 'pattern_counts': {'":': 7, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '317520', 'license': 'MIT', 'n_tokens_mistral': 2331, 'n_tokens_neox': 2247, 'n_words': 434, 'path': 'notebooks/01.0-custom-parsing/12.0-mobysound-whale-custom-parsing.ipynb', 'repo_name': 'xingjeffrey/avgn_paper', 'size': '7096'}