<commit_before>import datetime
import itertools
import dl
import xypath
import messytables
import orm
"""Value: dsID, region, indID, period, value, source, is_number
   DataSet: dsID, last_updated, last_scraped, name
   Indicator: indID, name, units
   """

url = 'http://cred01.epid.ucl.ac.be:5317/?after=&before=&agg1=iso&agg2=year&dl=true'
targets = 'no_disasters no_killed no_injured no_affected no_homeless total_affected total_dam'.split(' ')
names = ["Number of disasters", "People killed in disasters", "People injured in disasters", "People affected by disasters", "People made homeless by disasters", "Total number of people affeccted by disasters", "Total cost of damage done by disasters"]
num_years = 10
raw = dl.grab(url)
m_tables = messytables.any.any_tableset(raw)
mt, = m_tables.tables
table = xypath.Table.from_messy(mt)
nowyear = str(datetime.datetime.now().year)
minyear = str(datetime.datetime.now().year-num_years)
country_cells = table.filter('iso').assert_one().fill(xypath.DOWN)
country_year_filter = country_cells.filter(lambda b: b.shift(xypath.RIGHT).value >= minyear
                                                 and b.shift(xypath.RIGHT).value != nowyear)
target_cells = table.filter(lambda b: b.value in targets)<commit_msg>command line: number of years<commit_after>import datetime
import itertools
import dl
import xypath
import messytables
import orm
import sys

"""Value: dsID, region, indID, period, value, source, is_number
   DataSet: dsID, last_updated, last_scraped, name
   Indicator: indID, name, units
   """
try:
    num_years = int(sys.argv[1])
except:
    print "invoke with number of years (1, 10)"
    raise

url = 'http://cred01.epid.ucl.ac.be:5317/?after=&before=&agg1=iso&agg2=year&dl=true'
targets = 'no_disasters no_killed no_injured no_affected no_homeless total_affected total_dam'.split(' ')
names = ["Number of disasters", "People killed in disasters", "People injured in disasters", "People affected by disasters", "People made homeless by disasters", "Total number of people affeccted by disasters", "Total cost of damage done by disasters"]
raw = dl.grab(url)
m_tables = messytables.any.any_tableset(raw)
mt, = m_tables.tables
table = xypath.Table.from_messy(mt)
nowyear = str(datetime.datetime.now().year)
minyear = str(datetime.datetime.now().year-num_years)
country_cells = table.filter('iso').assert_one().fill(xypath.DOWN)
country_year_filter = country_cells.filter(lambda b: b.shift(xypath.RIGHT).value >= minyear
                                                 and b.shift(xypath.RIGHT).value != nowyear)
target_cells = table.filter(lambda b: b.value in targets)
{'fraction_non_alphanumeric': 0.11811621724268895, 'fraction_numerical': 0.008355488036460312, 'mean_word_length': 4.751091703056769, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '543466', 'n_tokens_mistral': 877, 'n_tokens_neox': 821, 'n_words': 224}