<jupyter_start><jupyter_text>Get Percentage for each catagory:<jupyter_code>ped_auto_model = BERTopic(nr_topics="auto")
ped_auto_model = BERTopic.load("ped_auto_model")
car_auto_model = BERTopic(nr_topics="auto")
car_auto_model = BERTopic.load("car_auto_model")
car_similar_topics, similarity = car_auto_model.find_topics("car", top_n=50)
print(car_similar_topics)
car_related_words_by_topic
ped_similar_topics, ped_similarity = ped_auto_model.find_topics("pedestrian", top_n=50)
print(ped_similar_topics)
ped_related_words_by_topic
df['224 - bus']
from collections import Counter
def count_dict_for_each_question(df, _related_words_by_topic, q=[]):
    num = 0
    q_count = 0
    q_portion = []
    f_col = ['Timestamp']
    count_dict = dict()
    for (column_name, column_data) in df.iteritems():
        col_values = []
        if column_name not in f_col:
            c_id, name = column_name.rsplit('-', 1)
            c_id = c_id.strip()
            c_id = int(c_id)

#             print(c_id)
#             print(ped_related_words_by_topic[c_id])
            for el in column_data:
                col_values.extend(list(set(el.split(';'))))
            ctr = Counter(col_values)
            
            o_flag = True
            for el in q:
                if el in ctr:
                    num = num + 1
                    q_count = q_count + ctr[el]
                    q_portion.append(ctr[el]/4)
                    o_flag = False
            if o_flag:
                q_portion.append(0)
                print(column_name)

#             print(ctr)
#             if 'pedestrian' in ctr:
#                 num = num + 1
#             elif 'Pedestrian' in ctr:
#                 num = num + 1
#             else:
#                 print(column_name)

            t_dict = dict()
            for el in _related_words_by_topic[c_id]:
                if el in ctr:
#                     print(el)
                    t_dict[el] = ctr[el]
                else:
                    t_dict[el] = 0
#             print(t_dict)
            count_dict[c_id] = t_dict
#             break
    print(num)
    print(q_count)
    return count_dict, q_portion
ped_count_dict, q_portion = count_dict_for_each_question(df, ped_temp_dict, q=['pedestrian', 'Pedestrian'])

# there is a pedestrian topic which is related to itself set it manually to 1
q_portion[0] = 1.0
print(q_portion)
print(len(q_portion))
sum(q_portion) / len(q_portion)
car_count_dict, q_portion = count_dict_for_each_question(c_df, car_temp_dict, q=['car', 'Car'])
# _count_dict
sum(q_portion) / len(q_portion)
157 / (4 * 50)
ped_count_dict
ped_all_ans = []
for el in ped_count_dict:
    for ct in ped_count_dict[el]:
        ped_all_ans.append(ped_count_dict[el][ct]/4)
sum(ped_all_ans) / len(ped_all_ans)
ped_count_dict[192]
car_count_dict
car_count_dict[153]
car_all_ans = []
for el in car_count_dict:
    for ct in car_count_dict[el]:
        car_all_ans.append(car_count_dict[el][ct]/4)
sum(car_all_ans) / len(car_all_ans)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10175090849025438, 'fraction_numerical': 0.011232243145028081, 'mean_word_length': 2.3946188340807173, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '343811', 'license': 'MIT', 'n_tokens_mistral': 1121, 'n_tokens_neox': 1058, 'n_words': 203, 'path': 'srv.ipynb', 'repo_name': 'AI-EnabledSoftwareEngineering-AISE/CADE', 'size': '3027'}