<jupyter_start><jupyter_text>* Count only highest ranking tag (what if both of them exist)* Bird tag is confusing, it can be also waterbird, how to handle ? Also Animal*<jupyter_code>#LOAD MODEL predictions
# splits_path= Path('/files/scratch/enis/data/nna/labeling/splits/')

# labelsbymodelpath=Path('/scratch/enis/data/nna/labeling/deep_labels/')
# dbfile = open(labelsbymodelpath / "classifications_all_2.pkl", 'rb')  
# modelresults = pickle.load(dbfile) 
# dbfile.close() 
# LOAD LABELS by human
labelsbyhumanpath=Path('/scratch/enis/data/nna/labeling/results/')
# filter by username
labelsbyhuman=[i for i in listdir(labelsbyhumanpath) if (".csv" in i) ]

humanresults={}
counter=0
for apath in labelsbyhuman:
    with open(labelsbyhumanpath / apath, newline='') as f:
        reader=csv.reader(f)
        for row in reader:
            counter+=1
            humanresults[row[0]]=row[1:]

print("unique files:",len(humanresults),"\ntotal files",counter)
#Join vehicle and Aircraft
for file_name,tagshere in humanresults.items():
#     print(file_name,tagshere)
    
    tagshere=["Aircraft" if tag == "Vehicle" else tag for tag in tags]
histogram={}
for file_name in humanresults:
    place_name=file_name.split("_")[0]
#     print(file_name)
    histogram.setdefault(place_name,0)
    histogram[place_name]+=1
a=list(histogram.items())
a.sort(key=lambda x:x[1],reverse=True)
# a
# load name of the labels
labels=load_labels()
items={}
from IPython.display import Audio,display,HTML
from ipywidgets import Output

items["mp3_output"]=Output()

def play_html_modify(mp3file,items={}):
#     out=items["mp3_output"]
#     displayed=display(HTML("<audio controls  loop autoplay><source src={} type='audio/mpeg'></audio>".format(mp3file)))
    displayed=display(HTML("<audio controls  loop><source src={} type='audio/mpeg'></audio>".format(mp3file)))

def vectorized_y_true(humanresults,tag_set):
    y_true={tag: [None]*len(humanresults) for tag in tag_set}
    for i,tags in enumerate(humanresults.values()):
        # we  only look for tags in tag_set
        for tag in tag_set:
            if tag in tags:
                y_true[tag][i] = 1
            else:
                y_true[tag][i] = 0 
    return y_true
y_true_dict=vectorized_y_true(humanresults,tag_set)

# for i,filename in enumerate(humanresults.keys()):
#     orig,audioop = modelresults[filename]

class color:
   PURPLE = '\033[95m'
   CYAN = '\033[96m'
   DARKCYAN = '\033[36m'
   BLUE = '\033[94m'
   GREEN = '\033[92m'
   YELLOW = '\033[93m'
   RED = '\033[91m'
   BOLD = '\033[1m'
   UNDERLINE = '\033[4m'
   END = '\033[0m'

names=["Bulbul"]
short_names=["Bulbul"]

def print_result(results):
    headers= ["Tag",]+short_names
    table=[]
#     sample_count=len(next(iter(y_true_dict.values())))
#     print("Total sample:",sample_count,"And threshold is",prob_threshold)
    for tag in (tag_set):
#         positive_count=sum(y_true_dict[tag])
        table.append([tag,*results[tag]])

    print(tabulate(table, headers=headers))

def reverse_results(results):
    results_clf2={}
    for tag,scores_per_Tag2 in results.items():
        scores_per_Tag=scores_per_Tag2[:]
        maximum=max(scores_per_Tag)
        scores_per_Tag=[color.BOLD + x + color.END if x==maximum else x for x in scores_per_Tag]

        for i,name in enumerate(names):
            score=scores_per_Tag[i]
            results_clf2.setdefault(name,[])
            results_clf2[name].append(score)      
    return results_clf2
        

def print_result2(results):
    results_clf=reverse_results(results)
    headers= ["Tag",]+list(tag_set)
    table=[]
#     sample_count=len(next(iter(y_true_dict.values())))
#     print("Total sample:",sample_count,"And threshold is",prob_threshold)
#     for tag in tag_set:
#         aresult=[]
    for i,name in enumerate(names):
#         positive_count=sum(y_true_dict[tag])
        table.append([short_names[i],*results_clf[name]])
    
    print(tabulate(table, headers=headers))

import csv 
csv_file="prediction_first--enis_10k--2019-10-06--21-37.csv"
data={}

with open(csv_file) as f:
    csv_reader = csv.reader(f)
    headers=next(csv_reader)
    for row in csv_reader:
        data[row[0]]=float(row[1])

# data.keys()
y_val_pred=[]
for path in humanresults:
    path2=path.replace(".mp3","")
    result=data[path2]
    y_val_pred.append(result)
np.empty_like(y_true_dict["Songbird"]).shape
from sklearn.model_selection import train_test_split

def warn(*args, **kwargs):
    pass
import warnings
warnings.warn = warn
from sklearn.metrics import roc_auc_score

results={}
for tag in (tag_set):
    y_true=y_true_dict[tag]
    
    X_train, X_test, y_train, y_test = train_test_split(
                    y_val_pred, y_true, test_size=0.2, random_state=42)
    X_train, X_val, y_train, y_val = train_test_split(
                    X_train, y_train, test_size=0.25, random_state=42)

#     score=roc_auc_score(y_true,y_val_pred)
# y_test,X_test
#     score=roc_auc_score(y_test,X_test)
# VALIDATION
#     score=roc_auc_score(y_val,X_val)
# TEST
    score=roc_auc_score(y_test,X_test)

    results[tag]=["{:.2}".format(score)]

# all_results[embed_type+"_"+map_reduce_type]=results
# print_result2(results)
# for embed_type in embed_types:
#     for map_reduce_type in map_reduce_embeds:
#         results=all_results[embed_type+"_"+map_reduce_type]
#         print("\n{} embeddings and merged by {} ".format(embed_type,map_reduce_type))
#         print_result2(results)
print_result2(results)
print_result2(results)
# tag="Songbird"
# X_train, X_test, y_train, y_test = train_test_split(
#     X_mean, y_true_dict[tag], test_size=0.2, random_state=42)
# X_train, X_val, y_train, y_val = train_test_split(
#     X_train, y_train, test_size=0.25, random_state=42)

# for name, clf in zip(names, classifiers):
# #         ax = plt.subplot(len(datasets), len(classifiers) + 1, i)
#     print(name)

#     try:
#         clf.fit(X_train, y_train)
#         y_val_pred=clf.predict_proba(X_val)
#         score=roc_auc_score(y_val,y_val_pred[:,1:])
#         print(y_val_pred[:10,1:])
#         print(clf.classes_)
# #         score = clf.score(X_val, y_val)
#     except:
#         print("ERROR")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13712267180475274, 'fraction_numerical': 0.01878612716763006, 'mean_word_length': 3.967304625199362, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '648079', 'license': 'MIT', 'n_tokens_mistral': 2340, 'n_tokens_neox': 2262, 'n_words': 412, 'path': 'notebooks/model_exp/Bulbul.ipynb', 'repo_name': 'EnisBerk/speech_audio_understanding', 'size': '6228'}