<jupyter_start><jupyter_text>Which modules use which attributes?__NOTE__: You should probably use the gradle script `./gradlew attributes` instead; that is how `attributes.json` was generated.<jupyter_code># https://stackoverflow.com/questions/14962485/finding-a-key-recursively-in-a-dictionary

def get_recursively(search_dict, field):
    """
    Takes a dict with nested lists and dicts,
    and searches all dicts for a key of the field
    provided.
    """
    fields_found = []

    for key, value in search_dict.items():

        if key == field:
            fields_found.append(value)

        elif isinstance(value, dict):
            results = get_recursively(value, field)
            for result in results:
                fields_found.append(result)

        elif isinstance(value, list):
            for item in value:
                if isinstance(item, dict):
                    more_results = get_recursively(item, field)
                    for another_result in more_results:
                        fields_found.append(another_result)

    return fields_found


module_files = [path.relative_to(MODULE_DIR) for path in Path(MODULE_DIR).rglob('*.json')]
attribute_module = {}
for mf in module_files:
    with open(MODULE_DIR/mf, encoding='utf-8') as fh:
        data = json.load(fh)
        for k in ['attribute', 'assign_to_attribute']:
            attribute_list = set(get_recursively(data, k))
            for a in attribute_list:
                if a not in attribute_module:
                    attribute_module[a] = []
                attribute_module[a].append(str(mf))

attribute_module<jupyter_output><empty_output><jupyter_text>What are the possible vlaues for each attribute?<jupyter_code>def get_attribute_values_recursively(search_dict):
    """
    Takes a dict with nested lists and dicts, searches all dicts and returns the value of the 'attribute' and 'value' fields if there is a field called 'attribute'
    """
    attribute_values = {} # key = attribute name, value = list (eventually set) of attribute values

    def record_attribute_value(attr, val):
        if attr not in attribute_values:
            attribute_values[attr] = set()
        # attribute_values[attr].append(val)
        if isinstance(val, set): # set
           attribute_values[attr].update(val)
        else:
           attribute_values[attr].add(val)

    if ('attribute' in search_dict) and ('value' in search_dict):
        record_attribute_value(search_dict['attribute'], search_dict['value'])
    
    for key, value in search_dict.items():

        if isinstance(value, dict):
            results = get_attribute_values_recursively(value)
            for a,v in results.items():
                record_attribute_value(a,v)

        if isinstance(value, list):
            for item in value:
                if isinstance(item, dict):
                    more_results = get_attribute_values_recursively(item)
                    for a,v in more_results.items():
                        record_attribute_value(a,v)

    return attribute_values
# mf = 'asthma.json'


attribute_values_list = []
for mf in module_files:
    # print(str(mf))
    with open(MODULE_DIR/mf, encoding='utf-8') as fh:
        data = json.load(fh)
        my_attribute_values = get_attribute_values_recursively(data)
        if len(my_attribute_values) > 0:
            attribute_values_list.append(my_attribute_values)



def aggregate_attribute_values(attr_val_list):
    attr_val_dict = {}
    for d in attr_val_list:
        for k,v in d.items():
            if k not in attr_val_dict:
                attr_val_dict[k] = set()
            attr_val_dict[k].update(v)
    return attr_val_dict

aggregate_attribute_values(attribute_values_list)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08970276008492568, 'fraction_numerical': 0.00291932059447983, 'mean_word_length': 2.6521317829457365, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '453217', 'license': 'MIT', 'n_tokens_mistral': 1119, 'n_tokens_neox': 1064, 'n_words': 275, 'path': 'system_data/get_attributes and_values_from_modules_json.ipynb', 'repo_name': 'rmhorton/virtual-generalist', 'size': '3768'}