<jupyter_start><jupyter_text>White munia custom parsing- An unlabelled dataset of white munia vocalizations-    - WAV files with individual ID- This notebook creates a JSON corresponding to each WAV file (and Noise file where available).- Dataset origin:    - https://www.ncbi.nlm.nih.gov/pubmed/24284561    - https://datadryad.org//resource/doi:10.5061/dryad.6pt8g<jupyter_code>from avgn.utils.general import prepare_env
prepare_env()<jupyter_output>env: CUDA_VISIBLE_DEVICES=GPU<jupyter_text>Import relevant packages<jupyter_code>from joblib import Parallel, delayed
from tqdm.autonotebook import tqdm
import pandas as pd
pd.options.display.max_columns = None
import librosa
from datetime import datetime
import numpy as np
import avgn
from avgn.custom_parsing.katahira_white_munia import generate_json
from avgn.utils.paths import DATA_DIR<jupyter_output><empty_output><jupyter_text>Load data in original format<jupyter_code>DATASET_ID = 'katahira_white_munia'
# create a unique datetime identifier for the files output by this notebook
DT_ID = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
DT_ID
DSLOC = avgn.utils.paths.Path('/mnt/cube/Datasets/WhiteMunia')
DSLOC
WAVLIST = list((DSLOC / 'WM').expanduser().glob('*/[!.]*.*')) 
len(WAVLIST), WAVLIST[0]
wav_df = pd.DataFrame(columns=['wavloc', 'indv', 'wav_num'])
for wf in WAVLIST:
    indv = wf.parent.stem
    wav_num = wf.stem
    wav_df.loc[len(wav_df)] = [wf, indv, wav_num]
print(len(wav_df))
wav_df[:3]<jupyter_output>177<jupyter_text>Generate JSON for wav<jupyter_code>for idx, row in tqdm(wav_df.iterrows(), total = len(wav_df)):
    generate_json(row, DT_ID)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11743341404358354, 'fraction_numerical': 0.012711864406779662, 'mean_word_length': 6.064102564102564, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '422221', 'license': 'MIT', 'n_tokens_mistral': 639, 'n_tokens_neox': 613, 'n_words': 133, 'path': 'notebooks/01.0-custom-parsing/14.0-white-munia-custom-parsing.ipynb', 'repo_name': 'xingjeffrey/avgn_paper', 'size': '1652'}