<jupyter_start><jupyter_text>Model Ensembling **This is a sample of how to use following Model_Ensembling classes** convert a file into a dictionary respresenting results from image captioning based model<jupyter_code>def cal_caption_result (caption_sim_wmd_file):
    lines_caption_sim = []
    with open(caption_sim_wmd_file) as f:
        lines_caption_sim = f.readlines()
    ar_cap_sim_dic={}
    for line in lines_caption_sim:
        segs=line.strip().split("\t")
        if segs[0] not in ar_cap_sim_dic:
            ar_cap_sim_dic[segs[0]]={}
        ar_cap_sim_dic[segs[0]][os.path.splitext(segs[1])[0]]= 1-float(segs[2])
    return ar_cap_sim_dic
ar_cap_sim_dic=cal_caption_result(r"result\test_caption_sim_wmd.txt")
print(len(ar_cap_sim_dic))<jupyter_output>1915<jupyter_text>convert a file into a dictionary respresenting results from face name matching based model<jupyter_code>def face_matching_result (face_matching_file):
    image_train_sim = []
    with open(face_matching_file) as f:
        image_train_sim = f.readlines()
    ar_train_sim_dic={}
    for line in image_train_sim:
        segs=line.strip().split("\t")
        if segs[0] not in ar_train_sim_dic:
            ar_train_sim_dic[segs[0]]=[]
        ar_train_sim_dic[segs[0]].append((os.path.splitext(segs[1])[0], segs[2], segs[3]))
    ar_train_sim_dic_cal={}
    
    for k, v in ar_train_sim_dic.items():
        if k not in ar_train_sim_dic_cal:
            ar_train_sim_dic_cal[k]={}
        for item in v:
            if int(item[2])==0:
                sim=0
            else:
                sim=(1-float(item[1]))*int(item[2])
            ar_train_sim_dic_cal[k][item[0]]= sim
    return ar_train_sim_dic_cal
ar_train_sim_dic_cal=face_matching_result(r"result\test_train_img_similarity.txt")
ar_crawl_sim_dic_cal=face_matching_result(r"result\test_crawl_img_similarity.txt")
print(len(ar_crawl_sim_dic_cal))<jupyter_output>191<jupyter_text>normalize value of a given dictionary (Min-max normalization)<jupyter_code>def norm_dict(a_dict):
    result={}
    amin, amax = min(a_dict.values()), max(a_dict.values())
    for k, v in a_dict.items():
        if amax-amin==0:
            result[k]=0
        else:
            result[k] = (v-amin) / (amax-amin)
    return result<jupyter_output><empty_output><jupyter_text>normalize value of a given dictionary the dictionary respresents results from face name matching based model<jupyter_code>def norm_sim(a_dict):
    result={}
    print(len(a_dict))
    for k, v in a_dict.items():
        result[k]=norm_dict(v)
    return result<jupyter_output><empty_output><jupyter_text>sort the value in the given dictionary<jupyter_code>def sort_dict(a_dict):
    normalized_dict=norm_sim(a_dict)
    result = {}
    for k, v in normalized_dict.items():
        sort_v = dict(sorted(v.items(), key=lambda item: item[1], reverse=True))
        result[k] = sort_v
    return result<jupyter_output><empty_output><jupyter_text>sort dictionary respresenting results from face name matching based modelsort dictionary respresenting results from image captioning matching based model<jupyter_code>cap_dict=sort_dict(ar_cap_sim_dic)
train_dict=sort_dict(ar_train_sim_dic_cal)
crawl_dict=sort_dict(ar_crawl_sim_dic_cal)<jupyter_output>1915
128
191<jupyter_text>merge results from image captioning based model and results from face matching based model<jupyter_code>def merge_cap_face(cap_dict, train_dict, crawl_dict, weight_cap, weight_img):
    result={}
    for k, v in cap_dict.items():
        img_id=os.path.splitext(k)[0]
        result[img_id]=v
        if k in train_dict:
            for k_tr, v_tr in train_dict[k].items():
                result[img_id][k_tr]=v_tr*weight_img
        if k in crawl_dict:
            for k_cr, v_cr in crawl_dict[k].items():
                result[img_id][k_cr]=v_cr*weight_img
    return result
cap_face_result=merge_cap_face(cap_dict, train_dict, crawl_dict, 0.5, 0.5)
sorted_cap_face_result=sort_dict(cap_face_result)<jupyter_output>1915<jupyter_text>truncate image candidates into top 100 list for each article<jupyter_code>def truncate_result(a_dict):
    for key, value in a_dict.items():
        l = [*value]
        l_100 = l[0:100]
        a_dict[key]=l_100
truncate_result(sorted_cap_face_result)
len(sorted_cap_face_result)<jupyter_output><empty_output><jupyter_text>acquire result from url matching based method<jupyter_code>img_id_name_dict = extract_img_url_token("../data/MediaEvalNewsImagesBatch04images.tsv",
                                              TEST_I_ID_IDX,
                                              TEST_IMG_URL_IDX)
article_id_name_dict = extract_article_token("../data/MediaEvalNewsImagesBatch04articles.tsv", 
                                             TEST_A_ID_IDX,
                                             TEST_TITLE_IDX)
url_result = match_url(article_id_name_dict, img_id_name_dict)<jupyter_output>matching url
1772
1772<jupyter_text>merge result from url matching based model into result<jupyter_code>def merge_url(final_cap_face_result, url_result):
    final_result={}
    for s_id, s_value in final_cap_face_result.items():
        if s_id in result:
            diff_elements = [x for x in url_result[s_id] if x not in s_value ]
            common_elements= [x for x in s_value if x in url_result[s_id] ]
            tail_elements=s_value[len(s_value)-len(diff_elements):]
            common_ele_in_tail=[x for x in common_elements if x in tail_elements]
            if len(common_ele_in_tail)>0:
                new_value=diff_elements+common_ele_in_tail+s_value[:len(s_value)-len(diff_elements)-len(common_ele_in_tail)]
            else:
                new_value=diff_elements+s_value[:len(s_value)-len(diff_elements)]
            final_result[s_id]= new_value

        else:
            final_result[s_id]=s_value
    return final_result
final_result=merge_url(sorted_cap_face_result, url_result)<jupyter_output><empty_output><jupyter_text>save final result into a file<jupyter_code>def save_final_result(output_file, final_result):
    with open(output_file, "w") as the_file:
        header="particleID"
        for i in range(100):
            header+="\t"+"iid"+str(i+1)
        the_file.write(header+"\n")
        for key, value in final_result.items():
            the_file.write(key+'\t'+ "\t".join(value)+"\n")
save_final_result(r"result\final_result.tsv", final_result)<jupyter_output><empty_output><jupyter_text>Result Exploring<jupyter_code>df=pd.read_csv(r"result\final_result.tsv", delimiter="\t")
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10013717421124829, 'fraction_numerical': 0.012193263222069806, 'mean_word_length': 3.7688953488372094, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '17657', 'license': 'MIT', 'n_tokens_mistral': 2382, 'n_tokens_neox': 2303, 'n_words': 407, 'path': 'notebooks/Model_Ensembling.ipynb', 'repo_name': 'minazhou2020/NewsImage', 'size': '6561'}