<jupyter_start><jupyter_text>Getting audio clips of required classes from google audioset data  https://research.google.com/audioset/dataset/index.html* Since the dataset is too huge, shortlisted a few classes to work on    * Natural sounds: Fire vs Wind/Water/Storm.* To get maximum data for the above classes, have used all the data available from the audioset data    * Refer https://research.google.com/audioset//download.htmlsplit for more details    * ID to Class mapping is available in this link - https://github.com/audioset/ontology/blob/master/ontology.json<jupyter_code>import os
import subprocess
import youtube_dl
import pandas as pd
import glob
import numpy as np
import tqdm
!pwd
os.chdir('/home/sramirez/git/FeuerFreiKiller/notebooks/')
!wget http://storage.googleapis.com/us_audioset/youtube_corpus/v1/csv/unbalanced_train_segments.csv -O '../data/external/unbalanced_data.csv'
!wget http://storage.googleapis.com/us_audioset/youtube_corpus/v1/csv/balanced_train_segments.csv -O '../data/external/balanced_train_data.csv'
!wget http://storage.googleapis.com/us_audioset/youtube_corpus/v1/csv/eval_segments.csv -O '../data/external/eval_data.csv'<jupyter_output>--2019-08-18 16:34:35--  http://storage.googleapis.com/us_audioset/youtube_corpus/v1/csv/eval_segments.csv
Resolving storage.googleapis.com (storage.googleapis.com)... 192.168.127.12, fc00:e968:6179::de52:7100
Connecting to storage.googleapis.com (storage.googleapis.com)|192.168.127.12|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1143389 (1,1M) [application/octet-stream]
Saving to: ‘../data/external/eval_data.csv’

../data/external/ev 100%[===================>]   1,09M  --.-KB/s    in 0,1s    

2019-08-18 16:34:36 (8,18 MB/s) - ‘../data/external/eval_data.csv’ saved [1143389/1143389]<jupyter_text>Merging the csv annotation files - Eval, Balanced and Unbalanced to get maximum data<jupyter_code>#os.chdir('../data/external/')
path = '../data/external/'
all_files = glob.glob(path + "/*.csv")
li = []

for filename in all_files:
    print(filename)
    df = pd.read_csv(filename, skiprows=2, quotechar='"', engine='python', skipinitialspace=True)
    li.append(df)

df = pd.concat(li, axis=0, ignore_index=True)
df.shape,  2042985 + 22176 + 20383
df.head()
df.duplicated().sum()
df['label'] = df.positive_labels.map(lambda x: 'single' if len(x.split(',')) == 1 else 'multi')
print(df.label.value_counts())
df.head()<jupyter_output><empty_output><jupyter_text>Filtering clips with required classes<jupyter_code># Use the corressponding ids based for the classes as available in json file mentioned at the start 
str2id = {'Fire': '/m/02_41', 'Wind': '/m/03m9d0z', 'Water': '/m/0838f', 'Thunderstorm': '/m/0jb2l'}
id2str = {v: k for k, v in str2id.items()}

for k in str2id.keys():
    df[k] = df.positive_labels.map(lambda x: 1 if (str2id[k] in x.split(',')) else 0)
for k in str2id.keys():
    print('Category: {}, # of elements: {}'.format(k, df[k].sum()))
    
list(str2id.keys())

# Filter out those registers with value for either classes considered (output class: fire)
columns = list(str2id.keys())
fw_df = df[df[columns].sum(axis=1) > 0]
fw_df.shape
fw_df.head()
# Translate after selection
fw_df['translated_labels'] = fw_df.positive_labels.map(lambda x: ','.join([id2str[y] for y in x.split(',') if y in id2str.keys()]))
fw_df.translated_labels.value_counts()
# Let's check an example for fire-water tuple
fw_df[fw_df.translated_labels == 'Fire,Water']
fw_df.head()
## Create subset with only around 1K from each type of sound, and all that mix several types.
size = 1100        # sample size
replace = False  # with replacement
subset = fw_df.groupby('translated_labels').apply(lambda x: x.sample(min(size, x.shape[0]), replace = replace)).reset_index(drop=True)
subset.translated_labels.value_counts()
fw_df.to_csv('../data/interim/firewind_dataset_links.csv')
fw_df.to_pickle('../data/interim/firewind_dataset_links.pickle')
subset.to_pickle('../data/interim/stratified_dataset_links.pickle')
subset.to_csv('../data/interim/stratified_dataset_links.csv')
assert(fw_df[fw_df['# YTID'] == "1icE61afXcY"].start_seconds.values[0] == 270)<jupyter_output><empty_output><jupyter_text>Download the 10 sec audio snippets for the filtered classes from youtube videos* Using ffmpeg to get the audio and extract the requried 10s clip* This part takes up lot of time as it involved downloading the entire video. Coudnt figure out a way to extract only the audio for a predefined time period. Any suggestions here would be very helpful<jupyter_code>meta_df = pd.read_pickle('../data/interim/stratified_dataset_links.pickle')
clipsmeta = list(zip(meta_df['# YTID'].values, meta_df.start_seconds.values))
meta_df.columns
meta_df.head() # positive labels can include other beyond nature sounds
meta_df.columns = ['YTID', 'start_seconds', 'end_seconds', 'positive_labels', 'label',
       'Water', 'Thunderstorm', 'Wind', 'Fire', 'translated_nature_labels']
meta_df.head()
len(clipsmeta)
processed_path = '../data/processed/'
os.chdir(processed_path)
!pwd
def download_audioset(metadata):
    i, start = metadata
    dur = 10
    #print(i, start, dur)
    ydl_opts = {'format': 'bestaudio/best',
                'outtmpl': './{}'.format(i+ '.mp4'),
                'postprocessors': [{'key': 'FFmpegExtractAudio','preferredcodec': 'wav','preferredquality': '192'}],
                'prefer_ffmpeg': True,
                'keepvideo': True,
                'info_dict': {'start_time': start,
                'end_time': start + dur}}
    try:
        with youtube_dl.YoutubeDL(ydl_opts) as ydl:
            #ydl.download(['http://www.youtube.com/watch?v={}'.format(i)]);
            url = 'http://www.youtube.com/watch?v={}'.format(i)
            #command = "ffmpeg -k -ss {} -t {} -i ./{}.wav ./{}.wav".format(start, dur, i,i+'_seg')
            command = "./ffmpeg -ss {} -i $(youtube-dl -f mp4 --get-url {}) -t {} -vn -c:v copy -c:a copy {}.aac".format(int(start), url, dur, i)
            print(command)
            subprocess.call(command, shell=True)
            #os.remove('./{}.wav'.format(i))
            #os.rename('./{}_seg.wav'.format(i),'./{}.wav'.format(i))
    except:
        print('Video not available{}'.format(i))
import multiprocessing as mp
mp.cpu_count()

with mp.Pool(8) as pool:
    list(tqdm.tqdm(pool.imap(download_audioset, clipsmeta), total=len(clipsmeta)))
# some clips will be missing due to unavailability of video
print(len(os.listdir('./')), len(clipsmeta))
# Let's see what is the most affected class

files = os.listdir('../data/processed/')
result = map(lambda x: x.split('.')[0], files)
saved_id = list(result)
not_present = set(meta_df.YTID).difference(set(saved_id))
len(not_present)
meta_df[meta_df.YTID.isin(not_present)].translated_nature_labels.value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15735595203275812, 'fraction_numerical': 0.026615969581749048, 'mean_word_length': 4.380802517702596, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '314982', 'license': 'MIT', 'n_tokens_mistral': 2524, 'n_tokens_neox': 2356, 'n_words': 545, 'path': 'notebooks/data_extraction.ipynb', 'repo_name': 'sramirez/FeuerFreiKiller', 'size': '6837'}