<jupyter_start><jupyter_text>California thrasher custom parsing- An labelled (but smaller) dataset of zebra finch vocalizations    - .WAV files with individual labels- This notebook creates a JSON corresponding to each WAV file (and Noise file where available).- Dataset origin:    - http://taylor0.biology.ucla.edu/birdDBQuery/<jupyter_code>from avgn.utils.general import prepare_env
prepare_env()<jupyter_output>env: CUDA_VISIBLE_DEVICES=GPU<jupyter_text>Import relevant packages<jupyter_code>from joblib import Parallel, delayed
from tqdm.autonotebook import tqdm
import pandas as pd
pd.options.display.max_columns = None
import librosa
from datetime import datetime
import numpy as np
import xlrd
import avgn
from avgn.custom_parsing.bird_db import generate_json
from avgn.utils.paths import DATA_DIR<jupyter_output><empty_output><jupyter_text>Load data in original format<jupyter_code>DATASET_ID = 'california_thrasher_cody'
# create a unique datetime identifier for the files output by this notebook
DT_ID = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
DT_ID
DSLOC = DATA_DIR/ 'raw' / 'bird-db' / 'CATH'
DSLOC
WAVLIST = list((DSLOC).expanduser().glob('*/wavs/*.wav'))
len(WAVLIST), WAVLIST[0]
song_db = pd.read_excel(DATA_DIR / "BIRD_DB.xls")
mainData_book = xlrd.open_workbook(DATA_DIR / "BIRD_DB.xls", formatting_info=True)
mainData_sheet = mainData_book.sheet_by_index(0)
song_urls = [
    ""
    if mainData_sheet.hyperlink_map.get((i, 11)) == None
    else mainData_sheet.hyperlink_map.get((i, 11)).url_or_path
    for i in range(mainData_sheet.nrows)
]
song_db["Audio_file"] = song_urls[1:]
song_db = song_db[1:]
song_db["file_stem"] = [
    i.split("/")[-1].split(".")[0] for i in song_db["Audio_file"].values
]
song_db[:3]<jupyter_output><empty_output><jupyter_text>Create a JSON for each wav<jupyter_code>for wavfile in tqdm(WAVLIST):
    generate_json(
            wavfile,
            DT_ID,
            song_db
        )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1133130081300813, 'fraction_numerical': 0.006097560975609756, 'mean_word_length': 5.331189710610932, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '569969', 'license': 'MIT', 'n_tokens_mistral': 733, 'n_tokens_neox': 706, 'n_words': 154, 'path': 'notebooks/01.0-custom-parsing/19.0-california-thrasher-custom-parsing.ipynb', 'repo_name': 'xingjeffrey/avgn_paper', 'size': '1968'}