<commit_before>"""
Municipality Profile data preparation
-------------------------------------
Gather data from the municipal finance API and provide values ready for display
on the page with little further processing.

If the API returns a null value, it can generally be treated as zero. That
happens in this library and nulls that should be zeros should not be
left to the frontend to handle.

The shape of data produced by this library is generally a series of years
or quarters in reverse chronological order with associated values. Only the
years that are to be shown are returned.

If data needed to calculate a value for a given date is missing, an
object is returned for that year with the value being None,
to indicate in the page that it is missing.
"""

from collections import defaultdict, OrderedDict
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from itertools import groupby
from requests_futures.sessions import FuturesSession
import dateutil.parser
import logging
import urllib

from utils import percent, ratio

logger = logging.getLogger('municipal_finance')

EXECUTOR = ThreadPoolExecutor(max_workers=10)

# The years for which we need results. Must be in desceneding order.
LAST_AUDIT_YEAR = 2016
LAST_AUDIT_QUARTER = '2016q4'
YEARS = list(xrange(LAST_AUDIT_YEAR-3, LAST_AUDIT_YEAR+1))
YEARS.reverse()

LAST_OPINION_YEAR = 2015
AUDIT_OPINION_YEARS = list(xrange(LAST_OPINION_YEAR-3, LAST_OPINION_YEAR+1))
AUDIT_OPINION_YEARS.reverse()

# we'll actually only have data up to the year before this but use four
# for consistency on the page.
LAST_UIFW_YEAR = 2015
UIFW_YEARS = list(xrange(LAST_UIFW_YEAR-3, LAST_UIFW_YEAR+1))
UIFW_YEARS.reverse()

YEAR_ITEM_DRILLDOWN = [
    'item.code',
    'financial_year_end.year',
]

<commit_msg>Advance to 2016 audit reports<commit_after>"""
Municipality Profile data preparation
-------------------------------------
Gather data from the municipal finance API and provide values ready for display
on the page with little further processing.

If the API returns a null value, it can generally be treated as zero. That
happens in this library and nulls that should be zeros should not be
left to the frontend to handle.

The shape of data produced by this library is generally a series of years
or quarters in reverse chronological order with associated values. Only the
years that are to be shown are returned.

If data needed to calculate a value for a given date is missing, an
object is returned for that year with the value being None,
to indicate in the page that it is missing.
"""

from collections import defaultdict, OrderedDict
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from itertools import groupby
from requests_futures.sessions import FuturesSession
import dateutil.parser
import logging
import urllib

from utils import percent, ratio

logger = logging.getLogger('municipal_finance')

EXECUTOR = ThreadPoolExecutor(max_workers=10)

# The years for which we need results. Must be in desceneding order.
LAST_AUDIT_YEAR = 2016
LAST_AUDIT_QUARTER = '2016q4'
YEARS = list(xrange(LAST_AUDIT_YEAR-3, LAST_AUDIT_YEAR+1))
YEARS.reverse()

LAST_OPINION_YEAR = 2016
AUDIT_OPINION_YEARS = list(xrange(LAST_OPINION_YEAR-3, LAST_OPINION_YEAR+1))
AUDIT_OPINION_YEARS.reverse()

# we'll actually only have data up to the year before this but use four
# for consistency on the page.
LAST_UIFW_YEAR = 2015
UIFW_YEARS = list(xrange(LAST_UIFW_YEAR-3, LAST_UIFW_YEAR+1))
UIFW_YEARS.reverse()

YEAR_ITEM_DRILLDOWN = [
    'item.code',
    'financial_year_end.year',
]


{'fraction_non_alphanumeric': 0.07071269487750557, 'fraction_numerical': 0.015033407572383074, 'mean_word_length': 4.7304625199362045, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '484667', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1066, 'n_words': 436}