<jupyter_start><jupyter_text>READ CSV Purpose :Reads csv from S2L Sound Component Training Data Collection (Combined) OR DEMO.csv and creates a class pkl containing ROI metadata  Notes :- Runs in ROI_generation conda env (envs/ROI_generation.yml)<jupyter_code># read csv 
import pandas as pd
import numpy as np
import csv
import joblib
import os 

import csv
import librosa
import matplotlib.pyplot as plt
import IPython.display as ipd

import joblib

import warnings
csv_path = "../data/demo_ROIs.csv" # ROI tags
df = pd.read_csv(csv_path, index_col=None,)
df<jupyter_output><empty_output><jupyter_text>The dataframe contains 5 S2L ROIs to generate mel specs and run through the modeling pipeline. Each 'Annotation' is a subclass of ABGQI:- OQU = quiet- AMU = anthrophony music- BBI = biophony bird- AMA = anthrophony machine- AVT = anthrophony vehicle traffic<jupyter_code># Number of wav files
len(df['File_name'].unique())
# average ROI length
length = df['End_Time_s'] - df['Begin_Time_s']
length.mean()
# ROI variance
length.var()
# Summary of ROI subtypes ('Annotation')
df.groupby(['Annotation']).sum()
# quick gathering of the class info (sub classes of ABGQI, here)
classes = []
cnt = "ROI_code"

# open csv
with open(csv_path,'rt')as f:
    data = csv.reader(f)
    next(data, None) # skip header
    
    # iterate through each row finding class annotations (e.g. AVT)
    for row in data:
        temp_class = row[8] # column == Annotation
 
        # aggregate classes
        if(temp_class != cnt):
            classes.append(temp_class)
            cnt = temp_class
# becasue there were duplicates!!
mylist = list(dict.fromkeys(classes))
print("There are", len(mylist), "classes")
classes = mylist
classes
# Gather ROI metadata
roi_list = []

# Iterate through each class
for i in range(len(classes)):
    roi_cnt = 0
    roi = classes[i]
    dur = 0 
    
    with open(csv_path,'rt') as f:
        data = csv.reader(f)
        next(data, None) # skip header
        
        for row in data:
            # ensure we have an annotation [8] with st [4] and stop [5] times
            if(row[8]==roi and row[5] != '' and row[4] != ''): 
                roi_cnt = roi_cnt+1
                dur = dur + (np.float16(row[5])-np.float16(row[4]))
    
    # if we have any valid ROIs save info in a list for each class          
    if(roi_cnt!=0)            :
        print(roi_cnt,'ROIs for',classes[i])
        print("Avg. duration: dur:",dur/roi_cnt)
        print('-'*50)
        roi_list.append([roi_cnt,classes[i]])
# count total ROIs and get names (metadata)
(roi_list)
roi_sum = 0
indx = 0
total_roi_w_cutoff = 0
class_name_lst = []
cutoff_class_name = []
cutoff = 1 # th nuo. or ROIs should at least be these many


for i in range(len(roi_list)):
    roi_sum = roi_sum + roi_list[i][0]
    
    if(roi_list[i][0] >=cutoff):
        indx+=1
        print(indx, roi_list[i])
        cutoff_class_name.append(roi_list[i])
        total_roi_w_cutoff += roi_list[i][0]
        class_name_lst.append(roi_list[i])
print("ROI_SUM with cutoff",total_roi_w_cutoff)
print("ROI_SUM for all classes",roi_sum)

# save class data
joblib.dump(cutoff_class_name, '../data/cutoff_class_name_demo.pkl')
print("Dumped")<jupyter_output>Dumped<jupyter_text>CLIP WAV FILES BASED ON ABOVE ROI PROCESSING<jupyter_code>dir_path = "../data/wavs/" # demo wavs
alt_dir_path = '' # possible other wav data
class_pkl = '../data/cutoff_class_name_demo.pkl'
os.path.exists(dir_path)
#load in class names, can skip script above if already exists
cutoff_class_name = joblib.load(class_pkl)
#warnings.filterwarnings('ignore')
sr = 22050 # sampling rate here

# iterate through each ROI class
for iii in range(len(cutoff_class_name)):
    excep = 0 # counting exceptions if any occurs
    print(cutoff_class_name[iii][1])
    class_name = cutoff_class_name[iii][1] # class of interest
    roi_temp = []
    roi_cnt = 0
        
    # open the ROI csv
    with open(csv_path,'rt')as f:
        data = csv.reader(f)
        next(data, None) 
        
        # iterate through each csv row
        for row in data:
            if(row[8] == class_name and row[0] != 'NA'):
                roi_cnt +=1
                print(class_name,":",roi_cnt)
                st = round((float(row[4]))*sr) #start time
                en = round((float(row[5]))*sr) #stop time

                y1 = round((float(row[6]))) # low freq
                y2 = round((float(row[7]))) # upper freq

                # read in wav file
                try:
                    x, sr = librosa.load(dir_path+row[0])
                    pth_ = dir_path+row[0]

                # try alternate wv dir
                except:
                    print("Main Dir doesn't have file... ")
                    print(dir_path+row[0] +"\n")
                    try:
                        x, sr = librosa.load(alt_dir_path+row[0])
                        pth_ = alt_dir_path+row[0]

                    except Exception as error:
                        print('***********************Caught this error: ' + repr(error))
                        break
                        excep +=1

                roi_temp.append((x[st:en],pth_,st,en, y1, y2))
    
    jl_file_dmp = os.path.join("../data/ROI_metadata/",cutoff_class_name[iii][1] +'.pkl')
    joblib.dump(roi_temp, jl_file_dmp)
    print('No. of ROIs for',cutoff_class_name[iii][1],'were:',len(roi_temp))
    print("Saved at",jl_file_dmp)
    print("Done with",cutoff_class_name[iii][1],'with',excep,'Exceptions.')
    print('-'*80)
# check for pickle dump success. Should match above total ROI counts
dir_path = '../data/ROI_metadata/'
dir_file_list = os.listdir(dir_path)

for i in range(len(dir_file_list)):
    jl = joblib.load(os.path.join(dir_path,dir_file_list[i]))
    print(np.shape(jl),dir_file_list[i].split('_')[0])

del jl<jupyter_output>(1, 6) AMA.pkl
(1, 6) AMU.pkl
(1, 6) AVT.pkl
(1, 6) BBI.pkl
(1, 6) OQU.pkl
{'fraction_non_alphanumeric': 0.12521093486331422, 'fraction_numerical': 0.012149848126898413, 'mean_word_length': 2.7229899497487438, 'pattern_counts': {'":': 1, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '196811', 'license': 'MIT', 'n_tokens_mistral': 2136, 'n_tokens_neox': 2041, 'n_words': 524, 'path': '0_melspec_generation-py/code/0_read_csv_create_ROIs.ipynb', 'repo_name': 'CQuinn8/ABGQI-CNN', 'size': '5926'}