<jupyter_start><jupyter_text>Testing vk approach<jupyter_code>import sys
sys.path.append("/home/ray__/ssd/BERT/")
sys.path.append("/home/ray__/CS/org/etherlabs/ai-engine/pkg/")
from gpt_feat_utils import GPT_Inference

#gpt_model = GPT_Inference("/home/ray__/ssd/BERT/models/se/epoch3/", device="cpu")
gpt_model = GPT_Inference("/home/ray__/ssd/BERT/models/ether/", device="cpu")

import pickle
import numpy as np

# ether
#ent_fv_full = pickle.load(open("/home/ether/hdd/ether/graph_dumps/se/se_entity_feats_v3.pkl", "rb"))
#com_map = pickle.load(open("mind_data/se/com_map_se_h_v3", "rb")`)
ent_fv_full = pickle.load(open("/home/ray__/ssd/minds/ether/ether_entity_feats_grp.pkl", "rb"))
com_map = pickle.load(open("/home/ray__/ssd/minds/ether/com_map.pkl", "rb"))
ranked_com = pickle.load(open("/home/ray__/ssd/minds/ether/ranked_com_selected.pkl","rb"))


    
common_entities = ent_fv_full.keys() & com_map.keys()
ent_fv = {}
for ent in common_entities:
    ent_fv[ent] = ent_fv_full[ent]


groups = pickle.load(open("/home/ray__/ssd/hinton/community_detection/group_segments/topic_testing/sync_call_today.pkl","rb"))
sent_fv = []
for group in groups:
    sent_fv_temp = []
    for sent in group:
        sent_fv_temp.append(gpt_model.get_text_feats(sent))
    sent_fv.append(np.mean(sent_fv_temp, axis=0))
from scipy.spatial.distance import cosine
ent_score = {}
for index, sent in enumerate(groups):
    ent_score[index] = []
    for ent in ent_fv.keys():
        ent_score[index].append( (ent, 1 - cosine(sent_fv[index], ent_fv[ent])))
        
keys_in_ent_score = ent_score.keys()
ent_score_sorted = {}
for segid in keys_in_ent_score:
    sorted_score = sorted(ent_score[segid], key=lambda x: x[1], reverse=True)
    ent_score_sorted[segid] = sorted_score    
group_ranked = {}
for groupid, ent_list in ent_score_sorted.items():
    print ("\n\n\nGroup Text:\n\n", groups[groupid])
    print("\nMost Similar Entities: ", [i for i,j in ent_list[:10]])
    group_ranked[groupid] = [ ranked_com[com_map[i]] for i,j in ent_list[:10] if i in com_map.keys() and com_map[i] in ranked_com.keys()]
for groupid, rank in sorted(group_ranked.items(), key= lambda kv:np.mean(kv[1]), reverse=False):
    print ("\n\n\nGroup Rank: ", rank ,"\n\n")
    print (*groups[groupid], sep="\n", end="\n\n")
    print("\nMost Similar Entities: ", [i for i,j in ent_list[:10]])
prev=-1
for ent, cls in sorted(com_map.items(), key=lambda kv:kv[1]):
    if prev!=cls:
        print ("\n\n------New Cluster-------\n\n")
        rrev=cls
    print (ent)
comb_ent = pickle.load(open("/home/ray__/ssd/minds/ether_se/se_ether_feats_updated.pkl","rb"))
query_ent = "Ether"
ms = []

for ent, fv in comb_ent.items():
    ms.append((ent, 1-cosine(fv, comb_ent[query_ent])))

sorted_ms = sorted(ms, key=lambda kv: kv[1], reverse=True)<jupyter_output><empty_output><jupyter_text>testing keyphrases<jupyter_code>text = """\
I was not able to test the IOS app dancing because of the same staging should that's going on so I tried testing it.
But right now I'm facing like when I do the call and if I want to watch it again, I'm getting a chalice error.
No, it's it's alive legitimate bundle app live and record.
Happens same with the latest Master Branch also.
Third look if I have tried to calls like after deployment like I got the transcript completely from the calls.
Mmm, and now like I'll just check whether it's my back but I'm short like it.
So let's string error from the master Branch as well.
But if I if Master branch is deployed, he cannot estimate transcript.
We'll do a deploy and test it against is a big plus time like master was working will deployed once again and check.
Oh, no it is because they promote more hits on the website and it was like because of the headset purposely like they were few memory issues and that's why like, uh, we were playing at us.
Otherwise, we answered the first part so that we will know whether it is getting the HTML file or not.
I was checking up on the website part of a lack of time.
"""
import sys
sys.path.append("/home/ray__/ssd/BERT/")
sys.path.append("/home/ray__/CS/org/etherlabs/ai-engine/pkg/")
sys.path.append("../")
from gpt_feat_utils import GPT_Inference

#gpt_model = GPT_Inference("/home/ray__/ssd/BERT/models/se/epoch3/", device="cpu")
gpt_model = GPT_Inference("/home/ray__/ssd/BERT/models/ether/", device="cpu")
from extra_preprocess import preprocess_text

text_list = [i for i in preprocess_text(text) if i!='']
text_fv = [gpt_model.get_text_feats(i) for i in text_list]
import networkx as nx
import numpy as np
import statistics
from scipy.spatial.distance import cosine
G = nx.Graph()

G.add_nodes_from(range(len(text_list)))

for nodea, fv1 in enumerate(text_fv):
    for nodeb, fv2 in enumerate(text_fv):
        G.add_edge(nodea, nodeb, weight=1 - cosine(fv1, fv2))
        
    closest_connection_n = sorted(dict(G[nodea]).items(), key=lambda kv:kv[1]["weight"], reverse=True)
    weights_n = list(map(lambda kv: (kv[1]["weight"]).tolist(), closest_connection_n))
    q3 = np.percentile(weights_n, 75)
    iqr = np.subtract(*np.percentile(weights_n, [75, 25]))
    outlier_score = q3 + iqr
    for nodeb, param in dict(G[nodea]).items():
        if param['weight']>=q3:
            pass
        else:
            G.remove_edge(nodea, nodeb)
from community import best_partition

com = best_partition(G, resolution=1)
com
com_map_local = {}
for key, cls in com.items():
    if cls in com_map_local.keys():
        com_map_local[cls].append(key)
    else:
        com_map_local[cls] = [key]
for cls in com_map_local.keys():
    content = [text_list[i] for i in com_map_local[cls]]
    print ("\n\n-------cluster-------\n")
    print (content)<jupyter_output>-------cluster-------

['I was not able to test the IOS app dancing because of the same staging should that is going on so I tried testing it.', 'Third look if I have tried to calls like after deployment like I got the transcript completely from the calls.', 'I was checking up on the website part of a lack of time.']


-------cluster-------

['But right now I am facing like when I do the call and if I want to watch it again, I am getting a chalice error.', 'No, it is it is alive legitimate bundle app live and record.', 'Oh, no it is because they promote more hits on the website and it was like because of the headset purposely like they were few memory issues and that is why like, uh, we were playing at us.']


-------cluster-------

['Mmm, and now like I will just check whether it is my back but I am short like it.', 'Otherwise, we answered the first part so that we will know whether it is getting the HTML file or not.']


-------cluster-------

['So let us string error from the mast[...]
{'fraction_non_alphanumeric': 0.1325283243887895, 'fraction_numerical': 0.005217650566487776, 'mean_word_length': 4.002982848620433, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '37512', 'license': 'MIT', 'n_tokens_mistral': 2367, 'n_tokens_neox': 2300, 'n_words': 725, 'path': 'community_detection/group_segments/generate_mind/recency_testing.ipynb', 'repo_name': 'etherlabsio/hinton', 'size': '6708'}