<commit_before>
import json
import urllib2
import os
from string import Template
from decimal import Decimal

# database stuff
from sqlalchemy.exc import DBAPIError
from sqlalchemy import or_
from climasng.models import *

from climasng.parsing.prosemaker import ProseMaker

class DocAssembler(object):

    def __init__(self, doc_data, section_data, settings={}):
        self._defaults = {
            # pattern can use region_id and region_type to make a url
            'region_url_pattern': 'http://localhost:6543/regiondata/${region_type}/${region_id}',
        }
        # merge in the user settings
        self._settings = dict(self._defaults.items() + settings.items())

        self._doc_data = doc_data
        self._sect_data = section_data
        self._region_type = doc_data['regiontype']
        self._region_id = doc_data['regionid']
        self._region_url_template = Template(self._settings['region_url_pattern'])
        self._region_data_path_template = Template(self._settings['region_data_path_pattern'])
        self._selected_sections = doc_data['selected_sections']
        self._format = doc_data['format_dest']
        self._year = doc_data['year']

        self._section_debug = self._settings.get('section_debug', False)

        self.getRegionData()
        self.getSource()


    def getRegionData(self, region_id=None, region_type=None):
        region_type = self._region_type if region_type is None else region_type
        region_id = self._region_id if region_id is None else region_id

        self._region = {
            'year': self._year,
            'region_type': region_type,
            'region_id': region_id
        }
        # resolve the region url template with region info
        self._region['region_url'] = self._region_url_template.substitute(self._region)
        self._region['region_data_path'] = self._region_data_path_template.substitute(self._region)

        json_string = urllib2.urlopen(self._region['region_url']).read()<commit_msg>fixed bug where the year was treated as a string<commit_after>
import json
import urllib2
import os
from string import Template
from decimal import Decimal

# database stuff
from sqlalchemy.exc import DBAPIError
from sqlalchemy import or_
from climasng.models import *

from climasng.parsing.prosemaker import ProseMaker

class DocAssembler(object):

    def __init__(self, doc_data, section_data, settings={}):
        self._defaults = {
            # pattern can use region_id and region_type to make a url
            'region_url_pattern': 'http://localhost:6543/regiondata/${region_type}/${region_id}',
        }
        # merge in the user settings
        self._settings = dict(self._defaults.items() + settings.items())

        self._doc_data = doc_data
        self._sect_data = section_data
        self._region_type = doc_data['regiontype']
        self._region_id = doc_data['regionid']
        self._region_url_template = Template(self._settings['region_url_pattern'])
        self._region_data_path_template = Template(self._settings['region_data_path_pattern'])
        self._selected_sections = doc_data['selected_sections']
        self._format = doc_data['format_dest']
        self._year = int(doc_data['year'])

        self._section_debug = self._settings.get('section_debug', False)

        self.getRegionData()
        self.getSource()


    def getRegionData(self, region_id=None, region_type=None):
        region_type = self._region_type if region_type is None else region_type
        region_id = self._region_id if region_id is None else region_id

        self._region = {
            'year': self._year,
            'region_type': region_type,
            'region_id': region_id
        }
        # resolve the region url template with region info
        self._region['region_url'] = self._region_url_template.substitute(self._region)
        self._region['region_data_path'] = self._region_data_path_template.substitute(self._region)

        json_string = urllib2.urlopen(self._region['region_url']).read()
{'fraction_non_alphanumeric': 0.09306930693069307, 'fraction_numerical': 0.0029702970297029703, 'mean_word_length': 3.280720338983051, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '645', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1155, 'n_words': 256}