<jupyter_start><jupyter_text>Explore indicator names to identify indicators that are not documented each year and indicators named differently in consecutive years, to support renaming or exclusion decisions.<jupyter_code>import pandas as pd
import os
import re
years=['2014','2015','2016','2017']
dirpath='//Users/rony/Dropbox/2_projects/Health_Geo/tables/'
#check indicator names
indicators=[]
filenames=[]
indicator_names={}
for year in years:
    input_path=dirpath+'{year}/prevalence/cleaned/'.format(year=year)
    filenames=os.listdir(path=input_path)
    filenames=[y for y in filenames if bool(re.search('.csv',y))==True] #clean none .csv files 
    
    indicators=[re.findall('(.*).csv',x)[0] for x in filenames]
    if '.ipynb_checkpoints' in filenames:
        filenames.remove('.ipynb_checkpoints')
    year_int=int(year)
    indicator_names[year]=indicators
#double iteration over the indicator list dictionary to identify names that differ for the same indicator
for year, indicators in indicator_names.items():
    #print (year,indicators)
    if int(year)<2017:
        year_1=str(int(year)+1)
        for year, indicators in indicator_names.items(): 
            indicators_1=indicator_names[year_1]
            if year!=year_1:
                set1=set (indicators)
                set2=set (indicators_1)
                if set1 - set2 !={}:
                    print ('indicator names difference , {year} and {year1}:'.format (year=year, year1=year_1), set1 - set2) 
                if set2 - set1 !={}:
                    print ('indicator names difference, {year1} and {year}:'.format (year=year, year1=year_1), set2 - set1)
                print ('-----------------')
#double iteration over the indicator list dictionary to identify names that differ for the same indicator
for year, indicators in indicator_names.items():
    #print (year,indicators)
    if int(year)<2017:
        year_1=str(int(year)+1)
        indicators_1=indicator_names[year_1]
        set1=set (indicators)
        
        set2=set (indicators_1)
        print ('indicator names difference , {year} and {year1}:'.format (year=year, year1=year_1), set1 - set2) 
        print ('indicator names difference, {year1} and {year}:'.format (year=year, year1=year_1), set2 - set1)
        print ()
        print ('indicators:', indicators)
        print ()

        print ('indicators_1:', indicators_1)
        print ()
        print ('-----------------')<jupyter_output>indicator names difference , 2014 and 2015: {'Learning_Disabilities_(18+)', 'Cardiovascular_Disease_-_Primary_Prevention', 'Hypothyroidism'}
indicator names difference, 2015 and 2014: {'Learning_Disabilities', 'Heart_Failure_Due_To_Left_Ventrical_Systolic_Dysfunction', 'Cardiovascular_Disease_-_Primary_Prevention_(30-74)', 'Obesity', 'Learning_Disabilities_All_Ages'}

indicators: ['Chronic_Obstructive_Pulmonary_Disease', 'Hypothyroidism', 'Mental_Health', 'Learning_Disabilities_(18+)', 'Cancer', 'Hypertension', 'Asthma', 'Coronary_Heart_Disease', 'Rheumatoid_Arthritis_(16+)', 'Osteoporosis_(50+)', 'Chronic_Kidney_Disease_(18+)', 'Dementia', 'Stroke_And_Transient_Ischaemic_Attack', 'Heart_Failure', 'Depression_(18+)', 'Cardiovascular_Disease_-_Primary_Prevention', 'Epilepsy_(18+)', 'Obesity_(16+)', 'Peripheral_Arterial_Disease', 'Atrial_Fibrillation', 'Palliative_Care', 'Diabetes_Mellitus_(17+)']

indicators_1: ['Chronic_Obstructive_Pulmonary_Disease', 'Mental_Health', 'Heart_Failure_Du[...]<jupyter_text>indicator names difference , 2014 and 2015: {'Learning_Disabilities_(18+)', 'Cardiovascular_Disease_-_Primary_Prevention', 'Hypothyroidism'}indicator names difference, 2015 and 2014: {'Learning_Disabilities', 'Heart_Failure_Due_To_Left_Ventrical_Systolic_Dysfunction', 'Cardiovascular_Disease_-_Primary_Prevention_(30-74)', 'Obesity', 'Learning_Disabilities_All_Ages'} Summary of manual check and decisions: indicators_names_differences.xsls<jupyter_code>decisions=pd.read_csv('indicators_names_differences.csv')
decisions<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12930393856824374, 'fraction_numerical': 0.031706712905622984, 'mean_word_length': 4.768571428571429, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '493772', 'license': 'MIT', 'n_tokens_mistral': 1411, 'n_tokens_neox': 1264, 'n_words': 272, 'path': 'preparation/data/prevalence/indicator_names.ipynb', 'repo_name': 'ronyarmon/geohealth', 'size': '4037'}