<commit_before>import re
import datetime
import requests
from orm import session, Value, DataSet, Indicator
import orm
import logging

logging.basicConfig()

"""Value: dsID, region, indID, period, value, source, is_number
   DataSet: dsID, last_updated, last_scraped, name
   Indicator: indID, name, units
   """

dsID = "data.undp.org"

dataset = {"dsID": dsID,
           "last_updated": None,  # TODO max(pubdate)
           "last_scraped": orm.now(), 
           "name": "UNDP Open Data"}
DataSet(**dataset).save()

data_url = "http://data.undp.org/resource/{}.json"

lookup = [{'soc':'ku9i-8fxp', 
           'fieldname':'gender_inequality_index_value_2013',
           'indID':'HDR:68606',
           'unit': 'index'},

          {'soc':'myer-egms',
           'fieldname':'_2013_gross_national_income_gni_per_capita_2011_ppp',
           'indID':'chd.eco.135',
           'unit': '2011 PPP $',
           'period': '2013'},

          {'soc':'xn26-t7qa',
           'fieldname':'expenditure_on_education_of_gdp_2005_2012',
           'indID':'PVE010',
           'unit': '% of GDP',
           'period': '2005-2012'},

          {'soc':'5tuc-d2a9',
           'fieldname':'mean_years_of_schooling',
           'indID':'PVE110',
           'unit': 'years'},

          {'soc':'5tuc-d2a9',
           'fieldname':'population_living_below_1_25_a_day',
           'indID':'PSE160',
           'unit': '%'},

          {'soc':'5tuc-d2a9',
           'fieldname':'expected_years_of_schooling',
           'indID':'PVE030',
           'unit':'years'},
          
          {'soc':'5tuc-d2a9',
           'fieldname':'maternal_mortality_ratio_deaths_per_100_000_live_births',
           'indID':'PVH180',
           'unit':'per 100,000 live births'},

          {'soc':'5tuc-d2a9',
           'fieldname':'adult_literacy_rate_ages_15_and_older',
           'indID':'PVE040',
           'unit':'%'},

          {'soc':'5tuc-d2a9',
           'fieldname':'impact_of_natural_disasters_number_of_deaths_per_year_per_million_people',
           'indID':'not_known',
           'unit':'per year per million people'},

          {'soc':'sf29-qtcx',
           'fieldname':'effects_of_environmental_threats_impact_of_natural_disasters_population_affected_per_year_per_million_people_2005_2012',
           'indID':'PVX070',
           'unit':'per year per million people',
           'period':'2005-2012'},

          {'soc':'myer-egms',
           'fieldname':'hdi_rank',
           'indID':'PSE220',
           'unit':'rank'},

          {'soc':'5tuc-d2a9',
           'fieldname':'under_five_mortality_rate',
           'indID': 'PVH120',
           'unit': 'per 100,000 live births'},

]

#lookup = {
#          OK "ku9i-8fxp": "HDR:68606",  # GII: Gender Inequality Index, value
          #OK "u2dx-####": "PSE110",  # GNI per capita in PPP terms (constant 2005 international $)
          #OK"bkr7-####": "PVE010",  # Public expenditure on education (% of GDP) (%)
          #OK"m67k-####": "PVE110",  # Mean years of schooling (of adults)|years
   #TODO       #"jbhn-####": "PVE120",  # Combined gross enrolment in education (both sexes)
          #OK"ehe9-####": "PSE160",  # MPI: Population living below $1.25 PPP per day (%)
          #???"a4ay-####": "PVH120",  # Under-five mortality
          #OK"qnam-####": "PVE030",  # Expected Year of Schooling (of children)
          #OK"4gkx-####": "PVH180",  # Maternal mortality ratio<commit_msg>correct units<commit_after>import re
import datetime
import requests
from orm import session, Value, DataSet, Indicator
import orm
import logging

logging.basicConfig()

"""Value: dsID, region, indID, period, value, source, is_number
   DataSet: dsID, last_updated, last_scraped, name
   Indicator: indID, name, units
   """

dsID = "data.undp.org"

dataset = {"dsID": dsID,
           "last_updated": None,  # TODO max(pubdate)
           "last_scraped": orm.now(), 
           "name": "UNDP Open Data"}
DataSet(**dataset).save()

data_url = "http://data.undp.org/resource/{}.json"

lookup = [{'soc':'ku9i-8fxp', 
           'fieldname':'gender_inequality_index_value_2013',
           'indID':'HDR:68606',
           'unit': 'Index'},

          {'soc':'myer-egms',
           'fieldname':'_2013_gross_national_income_gni_per_capita_2011_ppp',
           'indID':'chd.eco.135',
           'unit': '2011 PPP $',
           'period': '2013'},

          {'soc':'xn26-t7qa',
           'fieldname':'expenditure_on_education_of_gdp_2005_2012',
           'indID':'PVE010',
           'unit': 'Percentage',
           'period': '2005-2012'},

          {'soc':'5tuc-d2a9',
           'fieldname':'mean_years_of_schooling',
           'indID':'PVE110',
           'unit': 'Years'},

          {'soc':'5tuc-d2a9',
           'fieldname':'population_living_below_1_25_a_day',
           'indID':'PSE160',
           'unit': 'Percentage'},

          {'soc':'5tuc-d2a9',
           'fieldname':'expected_years_of_schooling',
           'indID':'PVE030',
           'unit': 'Years'},
          
          {'soc':'5tuc-d2a9',
           'fieldname':'maternal_mortality_ratio_deaths_per_100_000_live_births',
           'indID':'PVH180',
           'unit':'Deaths per 100,000 live births'},

          {'soc':'5tuc-d2a9',
           'fieldname':'adult_literacy_rate_ages_15_and_older',
           'indID':'PVE040',
           'unit':'Percentage'},

          {'soc':'5tuc-d2a9',
           'fieldname':'impact_of_natural_disasters_number_of_deaths_per_year_per_million_people',
           'indID':'not_known',
           'unit':'People affected per 1,000,000 people per year'},

          {'soc':'sf29-qtcx',
           'fieldname':'effects_of_environmental_threats_impact_of_natural_disasters_population_affected_per_year_per_million_people_2005_2012',
           'indID':'PVX070',
           'unit':'People affected per 1,000,000 people per year',
           'period':'2005-2012'},

          {'soc':'myer-egms',
           'fieldname':'hdi_rank',
           'indID':'PSE220',
           'unit':'Index'},

          {'soc':'5tuc-d2a9',
           'fieldname':'under_five_mortality_rate',
           'indID': 'PVH120',
           'unit': 'Deaths per 1,000 live births'},

]

#lookup = {
#          OK "ku9i-8fxp": "HDR:68606",  # GII: Gender Inequality Index, value
          #OK "u2dx-####": "PSE110",  # GNI per capita in PPP terms (constant 2005 international $)
          #OK"bkr7-####": "PVE010",  # Public expenditure on education (% of GDP) (%)
          #OK"m67k-####": "PVE110",  # Mean years of schooling (of adults)|years
   #TODO       #"jbhn-####": "PVE120",  # Combined gross enrolment in education (both sexes)
          #OK"ehe9-####": "PSE160",  # MPI: Population living below $1.25 PPP per day (%)
          #???"a4ay-####": "PVH120",  # Under-five mortality
          #OK"qnam-####": "PVE030",  # Expected Year of Schooling (of children)
          #OK"4gkx-####": "PVH180",  # Maternal mortality ratio
{'fraction_non_alphanumeric': 0.1675776754890679, 'fraction_numerical': 0.054372842347525895, 'mean_word_length': 2.5456399796022438, 'pattern_counts': {'":': 26, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7006060', 'n_tokens_mistral': 2750, 'n_tokens_neox': 2325, 'n_words': 413}