<jupyter_start><jupyter_text>calculate clipping for samples to be labeled* save them into a single dict and store as pkl<jupyter_code>split_Dir="/scratch/enis/data/nna/labeling/splits_real"
split_Dir="/scratch/enis/data/nna/labeling/splitsv3"

listOfFiles=list_files(split_Dir)
listOfFiles[0],len(listOfFiles)
area="splitsv3_clipping"
clipping_threshold=1
clipping_results_path="/scratch/enis/data/nna/labeling/"

# for i,area in enumerate(selected_areas):
#     area_filtered=file_properties_df[file_properties_df.site_id==area]
run_task_save(listOfFiles,area,clipping_results_path,clipping_threshold)

clippingFile="/scratch/enis/data/nna/labeling/splitsv3_clipping_1.pkl"

import numpy as np
clippingDict = np.load(clippingFile,allow_pickle=True)
clippingDict=clippingDict[()]


# split_Dir
# thePath="/scratch/enis/data/nna/labeling/splits_real/"

clippingThreshold=0.4
goodOnes=[]
for f in listOfFiles:
    left,right=clippingDict[f][0]
    maxValue=max(left,right)
    if maxValue<=clippingThreshold:
        goodOnes.append(f)

print(len(goodOnes),len(goodOnes)/len(listOfFiles))
    
print("a")
dst=Path("/scratch/enis/data/nna/labeling/splitsv3NoClipping04/")
dst.mkdir(exist_ok=True)
from shutil import copyfile

for src in goodOnes:
    src=Path(src)
    dst = src.parents[1]/ "splitsv3NoClipping04" / (src.stem+src.suffix)
    try:
        copyfile(src, dst)
    except Exception as e: print(e)  
!/home/enis/sbin/rclone copy /scratch/enis/data/nna/labeling/splitsv3NoClipping04/ CunyDropbox:server/data/nna/labeling/splitsv3NoClipping04/
!conda run -n speechEnv python /home/enis/projects/nna/src/slack_message.py -m "upload ended" 

!rm -rf /scratch/enis/data/nna/labeling/splitsv3/<jupyter_output><empty_output><jupyter_text>experiments for deciding clipping threshold<jupyter_code># load data, 10k 10 second long samples from real dataset
results_1=[]
results_99=[]
monoresults_1=[]
monoresults_99=[]
t1=time.time()
for ii,i in enumerate(split_files):
    try:
#         y, sr = librosa.load(i,mono=False,sr=None)
        y,sr=load_audio(i,dtype=np.int16,backend="pydub")
    except:
        print("error",i)
        pass

    # 1.0
    res=get_clipping_percent(y,1)
    if len(res)!=2:
        monoresults_1.append((i,res))
    else:
        results_1.append((i,res))
    
    #0.99
    res=get_clipping_percent(y,0.999)
    if len(res)!=2:
        monoresults_99.append((i,res))
    else:
        results_99.append((i,res))
    
    if ii%100==0:
        print(ii//100)
# sort results
results_1=sorted(results_1,key=lambda x:x[1][1])
print("worst",results_1[-1])

results_99=sorted(results_99,key=lambda x:x[1][1])
print("worst",results_99[-2])
# create numpy array
results_1np=np.array([i[1] for i in results_1])
results_99np=np.array([i[1] for i in results_99])
audiofiles=list_files("/home/enis/projects/nna/notebooks/trap_photo_audio_new3")

for i in range(10):
    res=get_clipping_percent(y[:,(y[0].shape[0]//10)*i:(y[0].shape[0]//10)*(i+1)],0.98)
    print(i,i+1,"{:.2}   {:.2}".format(*res))
# ipd.Audio(y[0], rate=sr)
np.sum([results_1np[:,1]<=0])
results_1np[2622]
# y, sr = librosa.load(Path(audiofiles[15]),mono=False,sr=None)
# ymono, sr = librosa.load(Path(audiofiles[15]),mono=True,sr=None)
sample=results_99[2800]
print(sample[1])
worst_path=sample[0]

y, sr = librosa.load(Path(worst_path),mono=False,sr=None)
ymono, sr = librosa.load(Path(worst_path),mono=True,sr=None)


# Generate data on commute times.
commutes = pd.Series(y[1])
# commutes = pd.Series(clipped)

commutes.plot.hist(cumulative=False,grid=True, bins=200, rwidth=0.999,
                   color='#617c8e',figsize=(9,4))
# commutes.plot()
plt.title('histogram of the signal')
plt.ylabel('Counts')
plt.xlabel('Amplitude')
plt.grid(axis='y', alpha=0.9)
# plt.savefig("cum_hist_1000bins.png",bbox_inches='tight',pad_inches=0.0)
# ipd.Audio(ymono, rate=sr)
# ipd.Audio(y[0], rate=sr)
# Plot a monophonic waveform
# y, sr = librosa.load(librosa.util.example_audio_file(), duration=10)
plt.figure(figsize=(12, 5))

plt.subplot(3, 1, 1)
librosa.display.waveplot(np.asfortranarray(y[0]), sr=sr)
plt.title('channel - 0')

# plt.figure(figsize=(12, 3))

plt.subplot(3, 1, 2)
librosa.display.waveplot(np.asfortranarray(y[1]), sr=sr)
plt.title('channel - 1')

plt.subplot(3, 1, 3)
librosa.display.waveplot(np.asfortranarray(ymono), sr=sr)
plt.title('Mono')
# plt.savefig('books_read.png')
# plt.show()
plt.tight_layout()
# plt.savefig('multiple.png')

# ipd.Audio(y[0], rate=sr)
# ipd.Audio(y[1], rate=sr)
"mono: {:.2}  cha1: {:.2}  cha2: {:.2}".format(*get_clipping_percent(y,1))

soundArray=np.array(y)
ynp.shape
np.sum(soundArray[0] >= 0.99) / np.sum(soundArray[0] > 0)
(np.sum(soundArray[0] <= -0.99))  /  np.sum(soundArray[0] < 0)
import warnings
warnings.filterwarnings("ignore")
len(np.where( ynp[0] > 0.9)[0])/2880000
# results=[]
# for i in audiofiles:
#     y, sr = librosa.load(i,mono=False,sr=None)
#     soundArray=np.array(y)
#     result=(((np.sum(soundArray[1] <= -0.99) + np.sum(soundArray[1] >= 0.99))) / soundArray[1].size)
#     results.append(result)
for audiof in audiofiles:
    y, sr = librosa.load(audiof,mono=False,sr=None)
    print("mono: {:.2}  cha1: {:.2}  cha2: {:.2}".format(*get_clipping_percent(y,1)))

split_files=list_files("/scratch/enis/data/nna/labeling/splits_real")

results_series=results_99np[:,1]
# results_series=results_series[results_series<=0.025]

# Generate data on commute times.
commutes = pd.Series(results_series)

commutes.plot.hist(cumulative=True,grid=True, bins=1000, rwidth=0.999,
                   color='#617c8e',figsize=(10,5))
plt.title('Clipping ratios for 10,000 samples')
plt.ylabel('Counts')
plt.xlabel('Clipping percentage')
plt.grid(axis='y', alpha=0.9)
plt.savefig("cum_hist_1000bins.png",bbox_inches='tight',pad_inches=0.0)
threshold=1
thresholdlowwer=0.2
results_1np[(results_1np[:,0]<=threshold) & (results_1np[:,1]<=threshold)].shape
upper=results_1np[(results_1np[:,0]<=threshold) & (results_1np[:,1]<=threshold)]
upper[(thresholdlowwer<upper[:,0]) & (thresholdlowwer<upper[:,1])].shape

threshold=0.2
thresholdlowwer=0.08
results_1np[(results_1np[:,0]<=threshold) & (results_1np[:,1]<=threshold)].shape
upper=results_1np[(results_1np[:,0]<=threshold) & (results_1np[:,1]<=threshold)]
upper[(thresholdlowwer<upper[:,0]) & (thresholdlowwer<upper[:,1])].shape

threshold=0.08
thresholdlowwer=0.02
upper=results_1np[(results_1np[:,0]<=threshold) & (results_1np[:,1]<=threshold)]
upper[(thresholdlowwer<upper[:,0]) & (thresholdlowwer<upper[:,1])].shape

threshold=0.02
thresholdlowwer=0.001

upper=results_1np[(results_1np[:,0]<=threshold) & (results_1np[:,1]<=threshold)]
upper[(thresholdlowwer<upper[:,0]) & (thresholdlowwer<upper[:,1])].shape

# 
threshold=0.001
# thresholdlowwer=0
upper=results_1np[(results_1np[:,0]<=threshold) & (results_1np[:,1]<=threshold)]
print(upper.shape)
# upper[(thresholdlowwer>upper[:,0]) & (thresholdlowwer>upper[:,1])].shape<jupyter_output>(2413, 2)
{'fraction_non_alphanumeric': 0.18194904275226717, 'fraction_numerical': 0.04663883690801785, 'mean_word_length': 6.155509783728116, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '189890', 'license': 'MIT', 'n_tokens_mistral': 2967, 'n_tokens_neox': 2827, 'n_words': 365, 'path': 'notebooks/explore/clipping_detection.ipynb', 'repo_name': 'EnisBerk/speech_audio_understanding', 'size': '6947'}