<jupyter_start><jupyter_text>Fix attrs for 5b6 and 5b9Since the generation and uploading of data mentioned before in task 6 are too nested, I want to fix it here, so the following code will "fix" the 5b6 and 5b9 features until it has the max level of 1.<jupyter_code>import git

git.Git("Downloads/").clone("https://github.com/chrismattmann/tika-similarity.git")
import json

j = json.load(open('Downloads/dsci-550-project_data.json'))
import pprint

j = {i:{k.replace('_', ' ').lower(): v for k,v in content.items()} for i, content in j.items()}
for i, content in j.items():
    if 'attacker estimated age' in content:
        try:
            # averge estimated age
            content['attacker estimated age'] = sum([float(v) for _, v in (content['attacker estimated age']['age prediction']).items()]) / len(content['attacker estimated age']['age prediction'])
        except:
            content['attacker estimated age'] = None
    
    if 'attacker relationship' in content:
        li = []
        
        mapping = {'Did I know your friend?': 'online', 'Did we meet in person before?': 'friend of a friend', 'Did we meet online?':'person'}
          
        for q, a in (content['attacker relationship']['boolean-questions']).items():
            if float(a['proba_yes']) > float(a['proba_no']):
                li.append(mapping[q])
                
        content['attacker relationship'] = li
                
    if 'name gender' in content:
        gender_li=[]
        for i, val in (content['name gender']['names']).items():
            if 'Gender' in val:
                gender_li.append(val['Gender'])
            else:
                gender_li.append(None)
                
        content['male count'] = gender_li.count('M')
        content['female count'] = gender_li.count('F')
        content['unknown count'] = gender_li.count(None)
        
        del content['name gender']
for i, content in j.items():
    pp.pprint(content)
    if int(i)>5: break
import requests

requests.put('https://copydsci550.firebaseio.com/.json', json=j)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13664596273291926, 'fraction_numerical': 0.008122312470138558, 'mean_word_length': 2.693121693121693, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '343326', 'license': 'MIT', 'n_tokens_mistral': 632, 'n_tokens_neox': 609, 'n_words': 185, 'path': 'notebooks/Task7-TikaSimilarity/Fix attrs.ipynb', 'repo_name': 'Anthonyive/DSCI-550-Assignment-1', 'size': '2093'}