<jupyter_start><jupyter_text>Swamp sparrow custom parsing- This dataset has:    - A number of CSVs with individual, element, and song information- The data was originally in [luscinia](https://rflachlan.github.io/Luscinia/) format (h2 db). I exported individual tables as CSVs so that I could import them into python. - Because the data is already well annotated, all this notebook does is save data as WAV and generate JSONs corresponding to the data. - Dataset origin:    - https://figshare.com/articles/SwampSparrow_luscdb_zip/5625310<jupyter_code>%load_ext autoreload
%autoreload 2
DATASET_ID = 'swamp_sparrow'
from avgn.utils.general import prepare_env
prepare_env()<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
env: CUDA_VISIBLE_DEVICES=GPU<jupyter_text>Import relevant packages<jupyter_code>from joblib import Parallel, delayed
from tqdm.autonotebook import tqdm
import pandas as pd
pd.options.display.max_columns = None
import librosa
from datetime import datetime
import numpy as np
import avgn
from avgn.custom_parsing.lachlan_swampsparrow import string2int16, annotate_bouts
from avgn.utils.paths import DATA_DIR<jupyter_output><empty_output><jupyter_text>Load data in original format<jupyter_code># create a unique datetime identifier for the files output by this notebook
DT_ID = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
DT_ID
DSLOC = avgn.utils.paths.Path('/mnt/cube/Datasets/swampsparrow/swampsparrow-xml/')
individual = pd.read_csv(DSLOC / 'INDIVIDUAL.csv')
individual[:3]
syllables = pd.read_csv(DSLOC / 'SYLLABLE.csv')
syllables[:3]
elements = pd.read_csv(DSLOC / 'ELEMENT.csv')
elements[:3]
elements.columns
len(elements)
import matplotlib.pyplot as plt
plt.plot([float(i) for i in elements.PEAKFREQ.values[10].split(' ')[:-1]])
plt.plot([float(i) for i in elements.HARMONICITY.values[0].split(' ')[:-1]])
songdata = pd.read_csv(DSLOC / 'SONGDATA.csv')
songdata[:3]
wavs = pd.read_csv(DSLOC / 'WAVS.csv')
wavs[:3]
len(wavs)
wavnames = [i[:-4] if '.wav' in i else i for i in wavs.FILENAME.values]
len(np.unique(wavnames))
labels = pd.read_excel(DATA_DIR / 'swamp_sparrow' / 'Classification160Final.xlsx', sheet_name='Classifications')
labels[:3]
wavnames2 = [i.split(',')[0] for i in labels[0.2628435522310395].values]
len(np.array(wavnames))
len(wavname == np.array(wavnames))
for wavname in wavnames2:
    wavlocs = np.where(wavname == np.array(wavnames))[0]
    if len(wavlocs)> 0:
        breakme
wavs.FILENAME.values[wavlocs[0]]
len(labels)
len<jupyter_output><empty_output><jupyter_text>generate wavs and textgrids<jupyter_code>"""with Parallel(n_jobs=1, verbose=10) as parallel:
    parallel(
        delayed(annotate_bouts)(
            row, 
            songdata[songdata.ID == row.ID].iloc[0], 
            individual[individual.ID == songdata[songdata.ID == row.ID].iloc[0].INDIVIDUALID].iloc[0], 
            elements[elements.SONGID == row.SONGID], 
            syllables[syllables.SONGID == row.SONGID], 
            DT_ID)
        for idx, row in tqdm(wavs.iterrows(), total=len(wavs))
    );
"""
"""for idx, row in tqdm(wavs.iterrows(), total=len(wavs)):
    annotate_bouts(
            row, 
            songdata[songdata.ID == row.ID].iloc[0], 
            individual[individual.ID == songdata[songdata.ID == row.ID].iloc[0].INDIVIDUALID].iloc[0], 
            elements[elements.SONGID == row.SONGID], 
            syllables[syllables.SONGID == row.SONGID], 
            DT_ID
    )"""<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12809564474807855, 'fraction_numerical': 0.016794762311414746, 'mean_word_length': 4.473520249221184, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '263829', 'license': 'MIT', 'n_tokens_mistral': 1326, 'n_tokens_neox': 1265, 'n_words': 273, 'path': 'notebooks/01.0-custom-parsing/03.0-swamp-sparrow-custom-parsing-Copy1.ipynb', 'repo_name': 'xingjeffrey/avgn_paper', 'size': '3513'}