<jupyter_start><jupyter_text>find the embedding for a given image<jupyter_code># p_path="/home/enis/projects/nna/data/cameraTraps/test.txt"

# with open(p_path) as p_file:
#     photo_paths=p_file.readlines()
#     photo_paths = [i.strip() for i in photo_paths]

# other_folder=["anwr_35_2019_101RECNX_RCNX3373.JPG",
# "anwr_42_2019_100RECNX_RCNX3574.JPG",
# "ivvavik_SINP03_2019_100RECNX_IMG_3219.JPG",
# "ivvavik_SINP05_2019_100RECNX_IMG_2867.JPG",
# "ivvavik_SINP06_2019_100RECNX_IMG_1884.JPG",
# "ivvavik_SINP09_2019_100RECNX_IMG_2743.JPG",
# "prudhoe_17_2019_100RECNX_RCNX3916.JPG",]

# parent_path1="/tank/data/nna/examples/randomPhotos10k/"
# parent_path2="/tank/data/nna/examples/randomPhotos1k/"

# # photo with Caribou
# photo_paths=['anwr_37_2019_100RECNX_RCNX9317.jpg']

# given image paths finds files, code is at:
# notebooks/explore/get_audio4photos.ipynb
file_properties_df=pd.read_pickle("../../data/realdata_v2No_stinchcomb.pkl")<jupyter_output><empty_output><jupyter_text>1. pick a sound to use for queries<jupyter_code># sound with Caribou

!find /scratch/enis/data/nna/real/ -iname "S4A10279_20190605_091602*"
# aircraft sound

!find /scratch/enis/data/nna/real/ -iname "S4A10255_20190507_073000*"<jupyter_output>^C<jupyter_text>1.1 find similar embeddings for given embeddings in the same location<jupyter_code># sound with Caribou

raw_embed_name="/scratch/enis/data/nna/real/anwr/37/2019/S4A10279_20190605_091602_vgg/S4A10279_20190605_091602_rawembeddings000.npy"
raw_embed=np.load(raw_embed_name)

# aircraft sound
# raw_embed_name="/scratch/enis/data/nna/real/prudhoe/26/2019/S4A10255_20190507_073000_vgg/S4A10255_20190507_073000_rawembeddings000.npy"
raw_embed_name="/scratch/enis/data/nna/real/prudhoe/26/2019/S4A10255_20190507_073000_vgg/S4A10255_20190507_073000_embeddings000.npy"

raw_embed=np.load(raw_embed_name)
raw_embed=raw_embed.astype(np.float32)

# S4A10255_20190507_073000-1808second
(30*60)+8<jupyter_output><empty_output><jupyter_text>1.2 Slice embedding for exact time of interest from a big file<jupyter_code># aircraft sound
start_seconds=(30*60)+8
raw_embed_audio=raw_embed[int(start_seconds):int(start_seconds)+60]

# # sound with Caribou
# start_seconds=(start_time-sorted_filtered["timestamp"])[0].total_seconds()
# raw_embed_audio=raw_embed[int(start_seconds):int(start_seconds)+60]
raw_embed_audio.shape<jupyter_output><empty_output><jupyter_text>2.1 load embeddings to search from<jupyter_code>%%time
# find all files
import glob
search_path="/scratch/enis/data/nna/real/prudhoe/26/"
filenamePattern="*_embeddings*.npy"

# all_embeddings=list_files("/scratch/enis/data/nna/real/anwr/37/",filenamePattern)
# aircraft sound
all_embeddings=list_files(search_path,filename=filenamePattern)
# remove original embedding from the list
# del all_embeddings[all_embeddings.index(raw_embed_name)]
# # len(all_embeddings)
# all_embeddings=[e for e in all_embeddings if "_embeddings" in e]<jupyter_output><empty_output><jupyter_text>2.1.1 this is for calculating distances ourself<jupyter_code>%%time
# load and merge embedding files
#LOAD:
embeds=[None]*len(all_embeddings)
index=0
for i,embed_file in enumerate(all_embeddings):
    embed=np.load(embed_file)
    index+=embed.shape[0]
    embeds[i]=(embed)
#MERGE:
# Faster then np.concatenate
concat_embeds=np.zeros([index,128],dtype=np.float32)
index=0
for i,embed in enumerate(embeds):
    concat_embeds[index:(index+embed.shape[0]),:]=embed[:]
    index+=embed.shape[0]
concat_embeds.shape<jupyter_output><empty_output><jupyter_text>2.1.2 create an ANNOY index for files at all_path on disk<jupyter_code>%%time
import time
# vector size
f=128
t = AnnoyIndex(f, 'euclidean')  # Length of item vector that will be indexed
# for i in range(concat_embeds.shape[0]):
#     t.add_item(i, concat_embeds[i,:])
#prepares annoy to build the index in the specified file instead of RAM 
#(execute before adding items, no need to save after build)
fn="/scratch/enis/data/nna/realMerged/prudhoe26_V11.ann"
t.on_disk_build(fn)
index=0
start=time.time()
for i,embed_file in enumerate(all_embeddings):
    embed=np.load(embed_file)
    for ii in range(embed.shape[0]):
        t.add_item(index, embed[ii,:])
        index+=1
    if i%1000==0:
        end=time.time()
        print(i//1000,end-start)
        start=end

t.set_seed(42)
t.build(16)<jupyter_output><empty_output><jupyter_text>3.1 Use Annoy NNs to approximate<jupyter_code>queries=[i for i in range(100,110,1)]
# QNNs=[[],[]]
Qmatrix=np.empty((len(queries),len(queries)))
sizeOfDataset=t.get_n_items()
for i,q in enumerate(queries):
    IDs,Distances=t.get_nns_by_item(q, 2,include_distances=True)
    IDs,Distances=IDs[1:],Distances[1:]
    Qmatrix[i,i]=Distances[0]
    for i2,q2 in enumerate(queries):
        if i==i2:
            continue
        else:
            alingedID=(IDs[0]+(i2-i))
            alingedID=0 if (alingedID)<0 else alingedID
            alingedID = alingedID if alingedID<sizeOfDataset else sizeOfDataset-1
            Qmatrix[i,i2]=t.get_distance(IDs[0],alingedID)


averageNNs_approx=np.average(Qmatrix,axis=1)
averageNNs_approx
# t.get_n_items()
len(all_path),len(all_embeddings)
!ls -alh /scratch/enis/data/nna/realMerged/allEmbeddings.ann
# concat_embeds.shape
# make sure it multiples of 10 to make things easy
concat_embeds=concat_embeds[:(concat_embeds.shape[0]//10)*10,:]<jupyter_output><empty_output><jupyter_text>there are three methods for creating vectors:<jupyter_code># # Method 1   1 second 
excerptLen=1
exp_name="1second"
concat_embeds2=concat_embeds[:]
raw_embed_audio2=raw_embed_audio[:]
raw_embed_audio2=raw_embed_audio2[20:21,:]
rowN=100000

#Method 2    mean 10 second
# excerptLen=10
# exp_name="Mean10Second"
# concat_embeds2=concat_embeds.reshape(-1,10,128).mean(axis=1)
# raw_embed_audio2=raw_embed_audio.reshape(-1,10,128).mean(axis=1)
# raw_embed_audio2=raw_embed_audio2[2:3,:]
# rowN=10000

# #Method 3    concat 10 second
# excerptLen=10
# exp_name="Concat10Second"
# concat_embeds2=concat_embeds.reshape(-1,1280)
# raw_embed_audio2=raw_embed_audio.reshape(-1,1280)
# raw_embed_audio2=raw_embed_audio2[2:3,:]
# rowN=10000


concat_embeds.shape
# concat_embeds3=concat_embeds[:,:]<jupyter_output><empty_output><jupyter_text>* we are trying to create a matrix, rows are queries and columns are data points in database* values are similarity, * then<jupyter_code>raw_embed_audio.shape

# paired_distances(raw_embed_audio, raw_embed_audio)
t.load()
# %%time
# t.build(10) # 10 trees

u = AnnoyIndex(f, "euclidian")
t.load(fn) # super fast, will just mmap the file
queryIndex=1000
a=(t.get_nns_by_item(queryIndex, 100)) # will find the 100 nearest neighbors
b=(t.get_nns_by_item(queryIndex+1, 100))

a=np.array(a)
b=np.array(b)

numbers=set()
for i in range(10):
    numbers=numbers.union(set(a).intersection(set(b+i)))
    numbers=numbers.union(set(a).intersection(set(b-i)))
len(numbers)
f=128
u = AnnoyIndex(f, 'euclidean')
u.load('/scratch/enis/data/nna/realMerged/prudhoe26.ann') # super fast, will just mmap the file

u.get_nns_by_item(1, 4)

u.get_n_items()
    allA=[None]*numberItems

    for i in range(numberItems):
        a=u.get_distance(i, index)
%%time
for m in range(10):
#     allA=[None]*concat_embeds.shape[0]
    for i in range(concat_embeds.shape[0]):
        a=u.get_distance(i, 0)
%%time
import multiprocessing

def worker(index,numberItems):
    allA=[None]*numberItems

    for i in range(numberItems):
        a=u.get_distance(i, index)
    return None

jobs = []
for index in range(10):
    numberItems=u.get_n_items()
    p = multiprocessing.Process(target=worker,args=(index,numberItems))
    jobs.append(p)
    p.start()

for p in jobs:
    p.join()
allA[4]
results[0]
%%time
# one sample/query
# res=paired_distances_broadcast(raw_embed_audio2,concat_embeds2)
# res.shape

#
# 10 queries
queryCount=10
startSecond=0
excerptLen=1

gamma=1/512
startIndex=int(startSecond/excerptLen)
endIndex=startIndex+queryCount


results = pairwise_distances(raw_embed_audio[startIndex:10,:], concat_embeds)

# results=np.exp(-results*gamma)


%%time
results=np.exp(-results*gamma)
# I might have to use this method, if I want to get maximum without replacement

# %%time
# stepSize=1
# windowSize=10
# resultsMax=[]
# resultsAvg=[]

# windowMax=np.max(results2,axis=0)

# for i in range(0,results2.shape[1],stepSize):
# #     windowElements=(results2[:,i:(i+windowSize+1)])
# #     colMax=np.max(windowElements,axis=0)

#     resultAvg=np.average(colMax)
#     resultsAvg.append(resultAvg)
# #     resultsMax.append(windowMax)<jupyter_output>CPU times: user 29.9 s, sys: 960 ms, total: 30.8 s
Wall time: 29.8 s<jupyter_text>Run sliding window over the Matrix<jupyter_code>%%time
windowSize=10
windowMax=np.max(results,axis=0)
windowMean=pd.Series(windowMax).rolling(window=windowSize).mean().iloc[windowSize-1:].values

%%time
windowSize=10
windowMax=np.max(resultMatrix,axis=0)
windowMean=pd.Series(windowMax).rolling(window=windowSize).mean().iloc[windowSize-1:].values<jupyter_output><empty_output><jupyter_text>Look into stats of results<jupyter_code>from scipy import stats

stats.describe(res)
res=windowMean[:]<jupyter_output><empty_output><jupyter_text>find kth highest scores (similarity of queries)<jupyter_code># arr = np.array([1, 3, 2, 4, 5,6,7,8,8,1,1,9])
kth=250
ind = np.argpartition(res, -kth)[-kth:]
sortedbyScore=sorted(list(zip(res[ind],ind)),reverse=True)<jupyter_output><empty_output><jupyter_text>Highest scores might be consequtive, merging these as one result helps decrease amount of files to listen<jupyter_code># find consequtive points that has high value 

theta=5
sortedbyInd=sorted(list(zip(res[ind],ind)),key=lambda x:x[1])
previous=sortedbyInd[0][1]
count=0
series=[]
starts=[]
for score,index in sortedbyInd:
    if index<=previous+theta:
        series.append((score,index))
    else:
        starts.append(series)
        series=[]
        series.append((score,index))
    previous=index

# sorted(starts,key=lambda x:x[1],reverse=True)
print("total periods",len(starts))
previous=0
print("short distances")
for score_indexes in starts:
    if (score_indexes[-1][1]-score_indexes[0][1])>30:
        print(score_indexes[0][1],score_indexes[-1][1])<jupyter_output><empty_output><jupyter_text>generating figures for similarity scores<jupyter_code>res.shape,rowN,res.shape[0]/rowN
import matplotlib.pyplot as plt
import matplotlib.ticker as plticker

minY,maxY=min(res),max(res)
exp_name="slidingWindow_AverageOfMax_short20k"
name=f"Euclidian Similarity {exp_name}"
nrows=res.shape[0]//rowN
print(nrows)
fig, ax = plt.subplots(nrows=nrows,figsize=(200, 20*nrows))
for i,axes in enumerate(ax):
    axes.plot(res[i*(rowN):(i+1)*(rowN)])
    axes.set_ylim(minY,maxY)
    axes.grid(True)

ax[0].set_ylabel(f"{name}",fontsize=32)

# plt.grid(True)

# loc = plticker.MultipleLocator(base=5000) # this locator puts ticks at regular intervals
# ax.xaxis.set_major_locator(loc)

plt.tight_layout()
fig.show()
filename=name.replace(" ","_")
fig.savefig(f"/home/enis/projects/nna/results/vis/nearestNeighbour/{filename}.png")

excerptLen=10
exp_name="Concat10Second"
concat_embeds2=concat_embeds.reshape(-1,1280)
raw_embed_audio2=raw_embed_audio.reshape(-1,1280)
raw_embed_audio2=raw_embed_audio2[2:3,:]
rowN=10000

res=paired_distances_broadcast(raw_embed_audio2,concat_embeds2)
res.shape

import matplotlib.pyplot as plt
import matplotlib.ticker as plticker

name=f"Euclidian Distance {exp_name}"
nrows=res.shape[0]//rowN

fig, ax = plt.subplots(nrows=nrows,figsize=(200, 20*nrows))
for i,axes in enumerate(ax):
    axes.plot(res[i*(rowN):(i+1)*(rowN)])

ax[0].set_ylabel(f"{name}",fontsize=32)

plt.grid(True)

# loc = plticker.MultipleLocator(base=5000) # this locator puts ticks at regular intervals
# ax.xaxis.set_major_locator(loc)

plt.tight_layout()
# fig.show()
filename=name.replace(" ","_")
fig.savefig(f"/home/enis/projects/nna/results/vis/nearestNeighbour/{filename}.png")

len(all_embeddings),excerptLen
# sorted([sorted(start) for start in starts])
split_folder="/home/enis/projects/nna/data/nearestNeighbours/S4A10255_20190507_073000-1808seconds/slidingWindow_AverageOfMax/"
excerptLen=1
# for indexes in (starts):
#     score,index=(indexes[len(indexes)//2])
#     print(score)
#     print(embedIndex2fileSecond(index,embeds,all_embeddings,excerptLen=1))
    
for score,index in (sortedbyScore):
#     score,index=(indexes[len(indexes)//2])
    print(score)
    print(embedIndex2fileSecond(index,embeds,all_embeddings,excerptLen=1))

# indexes=[indexes[len(indexes)//2] for indexes,count in (starts)]
# # # indexes
# clipAddBip(indexes,exp_name,split_folder,embeds,all_embeddings,excerptLen=10,reductionLen=1)
    

kdt = KDTree(concat_embeds2, leaf_size=30, metric='euclidean')
distances,indexes=kdt.query(raw_embed_audio2, k=10, return_distance=True)
allResults=[ (distance,index) for index,distance in (zip(indexes.flatten(),distances.flatten()))]
allResults.sort()
# distances,indexes=allResults
# distances
def embedIndex2fileSecond(index,embeds,all_embeddings,excerptLen=1):
    alist=[]
    countStart=0
    embed_count=index*excerptLen
    for i,em in enumerate(embeds):
        countEnd=countStart+(em.shape[0])
        if countStart<=embed_count and embed_count<=countEnd:
            startSecond=embed_count-countStart
    #             print(i,countStart,countEnd,"start second:",startSecond)
            filename=("/tank/"+"/".join(all_embeddings[i].split("/")[3:-2])+"/"+all_embeddings[i].split("/")[-2][:-4]+".flac")
#             alist.append((filename,startSecond))
            return((filename,startSecond))
#             break
        countStart=countEnd

    
split_folder="/home/enis/projects/nna/data/nearestNeighbours/"
def clipAddBip(allIndexes,exp_name,split_folder,embeds,all_embeddings,excerptLen,reductionLen):
    # embed_count=823804
    beep_wav="/home/enis/projects/nna/data/beep.wav"
    beep_wav = AudioSegment.from_file(beep_wav)
    beep_wav=beep_wav-30

    split_folder=(split_folder+exp_name)
    Path(split_folder).mkdir(parents=True, exist_ok=True)

    for index in allIndexes[:]:
        (filename,startSecond)=embedIndex2fileSecond(index,embeds,all_embeddings,excerptLen=reductionLen)
        endSecond=startSecond+excerptLen
        print(filename,startSecond)
        input_file=filename
        buffer=10
        start_time="{}.{}".format((startSecond-buffer)//60,(startSecond-buffer)%60)
        end_time="{}.{}".format((endSecond+buffer)//60,(endSecond+buffer)%60)
        outputSuffix=".wav"
        output_file = splitmp3(input_file,split_folder,start_time,end_time,depth=1,backend="ffmpeg",outputSuffix=outputSuffix)

        other_wav=AudioSegment.from_file(output_file)
        start,end=(buffer*1000),((buffer+excerptLen)*1000)
        result_wav=other_wav[:start]+(beep_wav)+other_wav[start:end]+beep_wav+other_wav[end:]
        o_file=result_wav.export(output_file)


# o_file=result_wav.export(output_file)
# len(other_wav)
# beep_wav="/home/enis/projects/nna/data/beep.wav"
# other_wav="./S4A10279_20190507_033000_4008-4068.wav"
# beep_wav = AudioSegment.from_file(beep_wav)
# beep_wav=beep_wav-30
# other_wav=AudioSegment.from_file(other_wav)
# other_wav[:start]+(beep_wav-30)+other_wav[start:end]+
# res=(beep_wav-30)+other_wav[:1000]

# input_file='/tank/data/nna/real/anwr/37/2019/S4A10279_20190605_091602.flac'
# split_folder="./"
# start_time="2.12"
# end_time="3.12"
# outputSuffix=".wav"
# a=splitmp3(input_file,split_folder,start_time,end_time,depth=1,backend="ffmpeg",outputSuffix=outputSuffix)


# !ffmpeg -i '/tank/data/nna/real/anwr/37/2019/S4A10279_20190605_091602.flac' -ss 304 -to 364 -c copy S4A10279_20190605_091602_304-364.wav


# !ffmpeg -i '/tank/data/nna/real/anwr/37/2019/S4A10279_20190604_234602.flac' -ss 650 -to 700 -c copy S4A10279_20190604_234602_650-700.wav


# !ffmpeg -i '/tank/data/nna/real/anwr/37/2019/S4A10279_20190509_073000.flac' -ss 4149 -to 4209 -c copy S4A10279_20190509_073000_4149-4209.wav

# !ffmpeg -i '/tank/data/nna/real/anwr/37/2019/S4A10279_20190530_214602.flac' -ss 3260 -to 3320 -c copy S4A10279_20190530_214602_3260-3320.wav

# !ffmpeg -i '/tank/data/nna/real/anwr/37/2019/S4A10279_20190507_033000.flac' -ss 4008 -to 4068 -c copy S4A10279_20190507_033000_4008-4068.wav

# !ffmpeg -i '/tank/data/nna/real/anwr/37/2019/S4A10279_20190507_033000.flac' -ss 320 -to 380 -c copy segment8.wav

# !ffmpeg -i '/tank/data/nna/real/anwr/37/2019/S4A10279_20190608_020000.flac' -ss 222 -to 282 -c copy S4A10279_20190608_020000_222-282.wav



# !mp3splt -d "./" -f -D /tank/data/nna/real/anwr/37/2019/S4A10279_20190517_130000.flac 15.27 16.27


# ffmpeg -i '/tank/data/nna/real/anwr/37/2019/S4A10279_20190605_091602.flac' -ss 304 -to 364 -c copy S4A10279_20190605_091602_304-364.wav


# !ls/tank/data/nna/real/anwr/37/2019/S4A10279_20190603_194602.flac<jupyter_output>books_read.png			get_audio4photos.ipynb
cum_hist_1000bins_025limit.svg	listen_photos.ipynb
cum_hist_1000bins.png		listen_segments_backup.ipynb
cum_hist_1000bins.svg		listen_segments.ipynb
cumulative_histogram2.svg	nearest_neighbors.ipynb
cumulative_histogram.jpg	Untitled1.ipynb
cumulative_histogram.png	Untitled.ipynb
{'fraction_non_alphanumeric': 0.13706146751503123, 'fraction_numerical': 0.07886288015877649, 'mean_word_length': 6.10576524263791, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '38031', 'license': 'MIT', 'n_tokens_mistral': 7710, 'n_tokens_neox': 6806, 'n_words': 938, 'path': 'notebooks/explore/nearest_neighbors-Annoy.ipynb', 'repo_name': 'EnisBerk/speech_audio_understanding', 'size': '17131'}